From 4c3c4555a35ec8e31ffbf3e96a5dba3bceec09ee Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 11 Jan 2024 11:18:52 +0800 Subject: [PATCH 001/147] fix: GroupApplicationAcceptedNotification --- pkg/rpcclient/notification/group.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rpcclient/notification/group.go b/pkg/rpcclient/notification/group.go index cbae7c49b2..f2413143d8 100755 --- a/pkg/rpcclient/notification/group.go +++ b/pkg/rpcclient/notification/group.go @@ -413,7 +413,7 @@ func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx conte if err := g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return err } - for _, userID := range append(userIDs, mcontext.GetOpUserID(ctx)) { + for _, userID := range append(userIDs, req.FromUserID) { err = g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationAcceptedNotification, tips) if err != nil { log.ZError(ctx, "failed", err) @@ -441,7 +441,7 @@ func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx conte if err := g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return err } - for _, userID := range append(userIDs, mcontext.GetOpUserID(ctx)) { + for _, userID := range append(userIDs, req.FromUserID) { err = g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationRejectedNotification, tips) if err != nil { log.ZError(ctx, "failed", err) From a00d77a7053c3f9999952337490f0493b4865c49 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 11 Jan 2024 16:31:24 +0800 Subject: [PATCH 002/147] fix: GroupApplicationAcceptedNotification --- pkg/rpcclient/notification/group.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/rpcclient/notification/group.go b/pkg/rpcclient/notification/group.go index f2413143d8..8c3719b2c1 100755 --- a/pkg/rpcclient/notification/group.go +++ b/pkg/rpcclient/notification/group.go @@ -409,11 +409,16 @@ func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx conte if err != nil { return err } - tips := &sdkws.GroupApplicationAcceptedTips{Group: group, HandleMsg: req.HandledMsg, ReceiverAs: 1} + tips := &sdkws.GroupApplicationAcceptedTips{Group: group, HandleMsg: req.HandledMsg} if err := g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return err } for _, userID := range append(userIDs, req.FromUserID) { + if userID == req.FromUserID { + tips.ReceiverAs = 0 + } else { + tips.ReceiverAs = 1 + } err = g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationAcceptedNotification, tips) if err != nil { log.ZError(ctx, "failed", err) @@ -442,6 +447,11 @@ func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx conte return err } for _, userID := range append(userIDs, req.FromUserID) { + if userID == req.FromUserID { + tips.ReceiverAs = 0 + } else { + tips.ReceiverAs = 1 + } err = g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationRejectedNotification, tips) if err != nil { log.ZError(ctx, "failed", err) From 48ff03f854e48fd22819c818586c6385d899a3c1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 12 Jan 2024 11:01:28 +0800 Subject: [PATCH 003/147] fix: NotificationUserInfoUpdate --- internal/rpc/group/group.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index f9b73ad2b3..abc271651f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -109,14 +109,11 @@ type groupServer struct { } func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgroup.NotificationUserInfoUpdateReq) (*pbgroup.NotificationUserInfoUpdateResp, error) { - defer log.ZDebug(ctx, "return") + defer log.ZDebug(ctx, "NotificationUserInfoUpdate return") members, err := s.db.FindGroupMemberUser(ctx, nil, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { - return nil, err - } groupIDs := make([]string, 0, len(members)) for _, member := range members { if member.Nickname != "" && member.FaceURL != "" { From 4266fed44f9f8dc9002634cb6cc6860a7d9f7434 Mon Sep 17 00:00:00 2001 From: withchao Date: Mon, 22 Jan 2024 03:13:27 +0000 Subject: [PATCH 004/147] cicd: robot automated Change --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/n_ws_server.go | 3 ++- internal/push/push_rpc_server.go | 3 ++- internal/push/push_to_client.go | 3 ++- internal/rpc/conversation/conversaion.go | 3 ++- internal/rpc/msg/as_read.go | 1 + internal/rpc/msg/seq.go | 1 + internal/rpc/third/s3.go | 6 ++++-- internal/rpc/user/callback.go | 1 + internal/rpc/user/user.go | 6 ++++-- pkg/common/convert/friend.go | 1 + pkg/common/db/controller/user.go | 9 ++++++++- pkg/common/db/mgo/friend.go | 1 + pkg/common/db/mgo/user.go | 13 +++++++++++-- pkg/common/db/table/relation/user.go | 3 ++- .../discoveryregister/kubernetes/kubernetes.go | 4 +++- pkg/rpcclient/msg.go | 1 + 18 files changed, 49 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 4332e389c2..739cf6b8e7 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/go-redis/redis v6.15.9+incompatible github.com/redis/go-redis/v9 v9.2.1 + github.com/spf13/pflag v1.0.5 github.com/stathat/consistent v1.0.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.45 go.uber.org/automaxprocs v1.5.3 @@ -116,7 +117,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/src-d/gcfg v1.4.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index d32a689cb5..659ff30369 100644 --- a/go.sum +++ b/go.sum @@ -15,11 +15,11 @@ cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/o cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= -github.com/AndrewZuo01/protocol v0.0.0-20240112093520-fd9c53e27b94 h1:o86vkek41ZrQqoBGqyKvS0z6N0uJj64mpzK72OkDZVM= -github.com/AndrewZuo01/protocol v0.0.0-20240112093520-fd9c53e27b94/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/IBM/sarama v1.41.3 h1:MWBEJ12vHC8coMjdEXFq/6ftO6DUZnQlFYcxtOJFa7c= github.com/IBM/sarama v1.41.3/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/OpenIMSDK/protocol v0.0.48 h1:8MIMjyzJRsruYhVv2ZKArFiOveroaofDOb3dlAdgjsw= +github.com/OpenIMSDK/protocol v0.0.48/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/tools v0.0.23 h1:xozfrGzhbpNPlDTap5DLVPk+JfgZ/ZyIj4Cuu3/bm9w= github.com/OpenIMSDK/tools v0.0.23/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 7e8129105f..01d92b92ac 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -19,7 +19,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/OpenIMSDK/tools/apiresp" "net/http" "os" "os/signal" @@ -29,6 +28,8 @@ import ( "syscall" "time" + "github.com/OpenIMSDK/tools/apiresp" + "github.com/go-playground/validator/v10" "github.com/redis/go-redis/v9" "golang.org/x/sync/errgroup" diff --git a/internal/push/push_rpc_server.go b/internal/push/push_rpc_server.go index 9e66f8f739..188ddc0e15 100644 --- a/internal/push/push_rpc_server.go +++ b/internal/push/push_rpc_server.go @@ -16,9 +16,10 @@ package push import ( "context" - "github.com/OpenIMSDK/tools/utils" "sync" + "github.com/OpenIMSDK/tools/utils" + "google.golang.org/grpc" "github.com/OpenIMSDK/protocol/constant" diff --git a/internal/push/push_to_client.go b/internal/push/push_to_client.go index 7cee7b99d3..ca90046058 100644 --- a/internal/push/push_to_client.go +++ b/internal/push/push_to_client.go @@ -18,9 +18,10 @@ import ( "context" "encoding/json" "errors" - "google.golang.org/grpc" "sync" + "google.golang.org/grpc" + "golang.org/x/sync/errgroup" "github.com/OpenIMSDK/protocol/constant" diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index d0d59547c5..40803089ca 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -17,9 +17,10 @@ package conversation import ( "context" "errors" - "github.com/OpenIMSDK/protocol/sdkws" "sort" + "github.com/OpenIMSDK/protocol/sdkws" + "github.com/OpenIMSDK/tools/tx" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index e91e2cf342..cb292421e9 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -18,6 +18,7 @@ import ( "context" utils2 "github.com/OpenIMSDK/tools/utils" + cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/redis/go-redis/v9" diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index c12f258b70..dfc2ad0b1b 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -16,6 +16,7 @@ package msg import ( "context" + pbmsg "github.com/OpenIMSDK/protocol/msg" ) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 2c230f258b..3b501d4add 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,12 +19,14 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/google/uuid" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" + "github.com/google/uuid" + + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/OpenIMSDK/protocol/third" diff --git a/internal/rpc/user/callback.go b/internal/rpc/user/callback.go index 092a66a07c..5276946a48 100644 --- a/internal/rpc/user/callback.go +++ b/internal/rpc/user/callback.go @@ -16,6 +16,7 @@ package user import ( "context" + pbuser "github.com/OpenIMSDK/protocol/user" "github.com/OpenIMSDK/tools/utils" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 240bea1533..e09c3299a1 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,12 +17,14 @@ package user import ( "context" "errors" - "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "math/rand" "strings" "time" + "github.com/OpenIMSDK/tools/pagination" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/OpenIMSDK/tools/tx" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" diff --git a/pkg/common/convert/friend.go b/pkg/common/convert/friend.go index 62ce6f95bb..27bd595ad5 100644 --- a/pkg/common/convert/friend.go +++ b/pkg/common/convert/friend.go @@ -17,6 +17,7 @@ package convert import ( "context" "fmt" + "github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/tools/utils" diff --git a/pkg/common/db/controller/user.go b/pkg/common/db/controller/user.go index 1a36510762..78ac5a7010 100644 --- a/pkg/common/db/controller/user.go +++ b/pkg/common/db/controller/user.go @@ -190,7 +190,14 @@ func (u *userDatabase) Page(ctx context.Context, pagination pagination.Paginatio func (u *userDatabase) PageFindUser(ctx context.Context, level1 int64, level2 int64, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) { return u.userDB.PageFindUser(ctx, level1, level2, pagination) } -func (u *userDatabase) PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) { + +func (u *userDatabase) PageFindUserWithKeyword( + ctx context.Context, + level1 int64, + level2 int64, + userID, nickName string, + pagination pagination.Pagination, +) (count int64, users []*relation.UserModel, err error) { return u.userDB.PageFindUserWithKeyword(ctx, level1, level2, userID, nickName, pagination) } diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index b4172d0fbb..851db61572 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" "go.mongodb.org/mongo-driver/mongo/options" diff --git a/pkg/common/db/mgo/user.go b/pkg/common/db/mgo/user.go index 7eed326349..34a25ed08e 100644 --- a/pkg/common/db/mgo/user.go +++ b/pkg/common/db/mgo/user.go @@ -16,10 +16,11 @@ package mgo import ( "context" + "time" + "github.com/OpenIMSDK/protocol/user" "github.com/OpenIMSDK/tools/errs" "go.mongodb.org/mongo-driver/bson/primitive" - "time" "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" @@ -89,7 +90,15 @@ func (u *UserMgo) PageFindUser(ctx context.Context, level1 int64, level2 int64, return mgoutil.FindPage[*relation.UserModel](ctx, u.coll, query, pagination) } -func (u *UserMgo) PageFindUserWithKeyword(ctx context.Context, level1 int64, level2 int64, userID string, nickName string, pagination pagination.Pagination) (count int64, users []*relation.UserModel, err error) { + +func (u *UserMgo) PageFindUserWithKeyword( + ctx context.Context, + level1 int64, + level2 int64, + userID string, + nickName string, + pagination pagination.Pagination, +) (count int64, users []*relation.UserModel, err error) { // Initialize the base query with level conditions query := bson.M{ "$and": []bson.M{ diff --git a/pkg/common/db/table/relation/user.go b/pkg/common/db/table/relation/user.go index 4039257f17..dbb2ff4647 100644 --- a/pkg/common/db/table/relation/user.go +++ b/pkg/common/db/table/relation/user.go @@ -16,9 +16,10 @@ package relation import ( "context" - "github.com/OpenIMSDK/protocol/user" "time" + "github.com/OpenIMSDK/protocol/user" + "github.com/OpenIMSDK/tools/pagination" ) diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discoveryregister/kubernetes/kubernetes.go index c105180568..06c58d9616 100644 --- a/pkg/common/discoveryregister/kubernetes/kubernetes.go +++ b/pkg/common/discoveryregister/kubernetes/kubernetes.go @@ -18,15 +18,17 @@ import ( "context" "errors" "fmt" - "github.com/stathat/consistent" "os" "strconv" "strings" + "github.com/stathat/consistent" + "google.golang.org/grpc" "github.com/OpenIMSDK/tools/discoveryregistry" "github.com/OpenIMSDK/tools/log" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" ) diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index abad0075ad..56167d7f44 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -17,6 +17,7 @@ package rpcclient import ( "context" "encoding/json" + "google.golang.org/grpc" "google.golang.org/protobuf/proto" From 7b5279a15b88e519290d7ff01a3a1ffa64e0898c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 13 Mar 2024 10:54:44 +0800 Subject: [PATCH 005/147] fix: component --- tools/component/component.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/component/component.go b/tools/component/component.go index 3220f36f73..e1f86e1207 100644 --- a/tools/component/component.go +++ b/tools/component/component.go @@ -99,13 +99,14 @@ func main() { } checks := []checkFunc{ - //{name: "Mysql", function: checkMysql}, {name: "Mongo", function: checkMongo, config: conf}, {name: "Redis", function: checkRedis, config: conf}, - {name: "Minio", function: checkMinio, config: conf}, {name: "Zookeeper", function: checkZookeeper, config: conf}, {name: "Kafka", function: checkKafka, config: conf}, } + if conf.Object.Enable == "minio" { + checks = append(checks, checkFunc{name: "Minio", function: checkMinio, config: conf}) + } for i := 0; i < maxRetry; i++ { if i != 0 { From 34daf1307f48e46be73c2c912a3c9d771a1c7eac Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 13 Mar 2024 14:37:03 +0800 Subject: [PATCH 006/147] fix: getConversationInfo --- internal/rpc/conversation/conversaion.go | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 91eec079bd..e78e4a18f0 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -506,19 +506,27 @@ func (c *conversationServer) getConversationInfo( switch chatLog.SessionType { case constant.SingleChatType: if chatLog.SendID == userID { - msgInfo.FaceURL = sendMap[chatLog.RecvID].FaceURL - msgInfo.SenderName = sendMap[chatLog.RecvID].Nickname + if recv, ok := sendMap[chatLog.RecvID]; ok { + msgInfo.FaceURL = recv.FaceURL + msgInfo.SenderName = recv.Nickname + } break } - msgInfo.FaceURL = sendMap[chatLog.SendID].FaceURL - msgInfo.SenderName = sendMap[chatLog.SendID].Nickname + if send, ok := sendMap[chatLog.SendID]; ok { + msgInfo.FaceURL = send.FaceURL + msgInfo.SenderName = send.Nickname + } case constant.GroupChatType, constant.SuperGroupChatType: - msgInfo.GroupName = groupMap[chatLog.GroupID].GroupName - msgInfo.GroupFaceURL = groupMap[chatLog.GroupID].FaceURL - msgInfo.GroupMemberCount = groupMap[chatLog.GroupID].MemberCount msgInfo.GroupID = chatLog.GroupID - msgInfo.GroupType = groupMap[chatLog.GroupID].GroupType - msgInfo.SenderName = sendMap[chatLog.SendID].Nickname + if group, ok := groupMap[chatLog.GroupID]; ok { + msgInfo.GroupName = group.GroupName + msgInfo.GroupFaceURL = group.FaceURL + msgInfo.GroupMemberCount = group.MemberCount + msgInfo.GroupType = group.GroupType + } + if send, ok := sendMap[chatLog.SendID]; ok { + msgInfo.SenderName = send.Nickname + } } pbchatLog.ConversationID = conversationID msgInfo.LatestMsgRecvTime = chatLog.SendTime From dc57d38856bfcfd7ef0e2ff6c9627530e59f5d81 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 25 Apr 2024 16:50:24 +0800 Subject: [PATCH 007/147] feat: cron task --- config/openim-crontask.yml | 4 +- go.mod | 9 +- internal/rpc/conversation/conversaion.go | 43 +++++ internal/rpc/msg/clear.go | 77 ++++++++ internal/tools/conversation.go | 149 --------------- internal/tools/cron_task.go | 111 ++++------- internal/tools/cron_task_test.go | 113 ------------ internal/tools/msg.go | 226 ----------------------- internal/tools/msg_doc_convert.go | 46 ----- pkg/authverify/token.go | 1 - pkg/common/config/config.go | 2 - pkg/common/db/controller/msg.go | 75 ++++++++ pkg/common/db/mgo/conversation.go | 11 +- pkg/common/db/mgo/msg.go | 86 +++++++++ pkg/common/db/mgo/msg_del.go | 71 +++++++ pkg/common/db/mgo/msg_test.go | 46 +++++ pkg/common/db/table/relation/msg.go | 6 + pkg/rpcclient/conversation.go | 5 + pkg/rpcclient/msg.go | 5 + 19 files changed, 470 insertions(+), 616 deletions(-) create mode 100644 internal/rpc/msg/clear.go delete mode 100644 internal/tools/conversation.go delete mode 100644 internal/tools/cron_task_test.go delete mode 100644 internal/tools/msg.go delete mode 100644 internal/tools/msg_doc_convert.go create mode 100644 pkg/common/db/mgo/msg_del.go create mode 100644 pkg/common/db/mgo/msg_test.go diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 39f730112a..9bbccfd252 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,4 +1,2 @@ -chatRecordsClearTime: "0 2 * * 3" -msgDestructTime: "0 2 * * *" +chatRecordsClearTime: "0 2 * * *" retainChatRecords: 365 -enableCronLocker: false diff --git a/go.mod b/go.mod index 891125a899..58354b107d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/openimsdk/open-im-server/v3 -go 1.20 +go 1.21.2 + +toolchain go1.21.9 require ( firebase.google.com/go v3.13.0+incompatible @@ -172,3 +174,8 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + + +replace ( + github.com/openimsdk/protocol => /Users/chao/Desktop/withchao/protocol +) \ No newline at end of file diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 1bf612923c..96d2a403f6 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -536,3 +536,46 @@ func (c *conversationServer) GetConversationNotReceiveMessageUserIDs(ctx context } return &pbconversation.GetConversationNotReceiveMessageUserIDsResp{UserIDs: userIDs}, nil } + +func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconversation.UpdateConversationReq) (*pbconversation.UpdateConversationResp, error) { + m := make(map[string]any) + if req.RecvMsgOpt != nil { + m["recv_msg_opt"] = req.RecvMsgOpt.Value + } + if req.AttachedInfo != nil { + m["attached_info"] = req.AttachedInfo.Value + } + if req.Ex != nil { + m["ex"] = req.Ex.Value + } + if req.IsPinned != nil { + m["is_pinned"] = req.IsPinned.Value + } + if req.GroupAtType != nil { + m["group_at_type"] = req.GroupAtType.Value + } + if req.MsgDestructTime != nil { + m["msg_destruct_time"] = req.MsgDestructTime.Value + } + if req.IsMsgDestruct != nil { + m["is_msg_destruct"] = req.IsMsgDestruct.Value + } + if req.BurnDuration != nil { + m["burn_duration"] = req.BurnDuration.Value + } + if req.IsPrivateChat != nil { + m["is_private_chat"] = req.IsPrivateChat.Value + } + if req.MinSeq != nil { + m["min_seq"] = req.MinSeq.Value + } + if req.MaxSeq != nil { + m["max_seq"] = req.MaxSeq.Value + } + if len(m) > 0 { + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.UserIDs, req.ConversationID, m); err != nil { + return nil, err + } + } + return &pbconversation.UpdateConversationResp{}, nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go new file mode 100644 index 0000000000..774eae32cd --- /dev/null +++ b/internal/rpc/msg/clear.go @@ -0,0 +1,77 @@ +package msg + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/wrapperspb" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "strings" + "time" +) + +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { + if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { + return nil, err + } + if req.Timestamp > time.Now().UnixMilli() { + return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") + } + var ( + docNum int + msgNum int + start = time.Now() + ) + clearMsg := func(ctx context.Context) (bool, error) { + conversationSeqs := make(map[string]struct{}) + defer func() { + req := &conversation.UpdateConversationReq{ + MsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), + } + for conversationID := range conversationSeqs { + req.ConversationID = conversationID + if err := m.Conversation.UpdateConversations(ctx, req); err != nil { + log.ZError(ctx, "update conversation max seq failed", err, "conversationID", conversationID, "msgDestructTime", req.MsgDestructTime) + } + } + }() + msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, 100) + if err != nil { + return false, err + } + if len(msgs) == 0 { + return false, nil + } + for _, msg := range msgs { + index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) + if err != nil { + return false, err + } + if len(index) == 0 { + return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") + } + docNum++ + msgNum += len(index) + conversationID := msg.DocID[:strings.LastIndex(msg.DocID, ":")] + if _, ok := conversationSeqs[conversationID]; !ok { + conversationSeqs[conversationID] = struct{}{} + } + } + return true, nil + } + for { + keep, err := clearMsg(ctx) + if err != nil { + log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + return nil, err + } + if !keep { + log.ZInfo(ctx, "clear msg success", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + break + } + log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + } + return &msg.ClearMsgResp{}, nil +} diff --git a/internal/tools/conversation.go b/internal/tools/conversation.go deleted file mode 100644 index 3e2a88ffd6..0000000000 --- a/internal/tools/conversation.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "context" - "math/rand" - "time" - - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/idutil" - "github.com/openimsdk/tools/utils/stringutil" -) - -// func (c *MsgTool) ConversationsDestructMsgs() { -// log.ZInfo(context.Background(), "start msg destruct cron task") -// ctx := mcontext.NewCtx(utils.GetSelfFuncName()) -// conversations, err := c.conversationDatabase.GetConversationIDsNeedDestruct(ctx) -// if err != nil { -// log.ZError(ctx, "get conversation id need destruct failed", err) -// return -// } -// log.ZDebug(context.Background(), "nums conversations need destruct", "nums", len(conversations)) -// for _, conversation := range conversations { -// ctx = mcontext.NewCtx(utils.GetSelfFuncName() + "-" + utils.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID) -// log.ZDebug( -// ctx, -// "UserMsgsDestruct", -// "conversationID", -// conversation.ConversationID, -// "ownerUserID", -// conversation.OwnerUserID, -// "msgDestructTime", -// conversation.MsgDestructTime, -// "lastMsgDestructTime", -// conversation.LatestMsgDestructTime, -// ) -// now := time.Now() -// seqs, err := c.msgDatabase.UserMsgsDestruct(ctx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) -// if err != nil { -// log.ZError(ctx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) -// continue -// } -// if len(seqs) > 0 { -// if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]interface{}{"latest_msg_destruct_time": now}); err -// != nil { -// log.ZError(ctx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) -// continue -// } -// if err := c.msgNotificationSender.UserDeleteMsgsNotification(ctx, conversation.OwnerUserID, conversation.ConversationID, seqs); err != nil { -// log.ZError(ctx, "userDeleteMsgsNotification failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) -// } -// } -// } -//} - -func (c *MsgTool) ConversationsDestructMsgs() { - log.ZInfo(context.Background(), "start msg destruct cron task") - ctx := mcontext.NewCtx(stringutil.GetSelfFuncName()) - num, err := c.conversationDatabase.GetAllConversationIDsNumber(ctx) - if err != nil { - log.ZError(ctx, "GetAllConversationIDsNumber failed", err) - return - } - const batchNum = 50 - log.ZDebug(ctx, "GetAllConversationIDsNumber", "num", num) - if num == 0 { - return - } - count := int(num/batchNum + num/batchNum/2) - if count < 1 { - count = 1 - } - maxPage := 1 + num/batchNum - if num%batchNum != 0 { - maxPage++ - } - for i := 0; i < count; i++ { - pageNumber := rand.Int63() % maxPage - pagination := &sdkws.RequestPagination{ - PageNumber: int32(pageNumber), - ShowNumber: batchNum, - } - conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) - if err != nil { - log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) - continue - } - log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) - if len(conversationIDs) == 0 { - continue - } - conversations, err := c.conversationDatabase.GetConversationsByConversationID(ctx, conversationIDs) - if err != nil { - log.ZError(ctx, "GetConversationsByConversationID failed", err, "conversationIDs", conversationIDs) - continue - } - temp := make([]*relation.ConversationModel, 0, len(conversations)) - for i, conversation := range conversations { - if conversation.IsMsgDestruct && conversation.MsgDestructTime != 0 && (time.Now().Unix() > (conversation.MsgDestructTime+conversation.LatestMsgDestructTime.Unix()+8*60*60)) || - conversation.LatestMsgDestructTime.IsZero() { - temp = append(temp, conversations[i]) - } - } - for _, conversation := range temp { - ctx = mcontext.NewCtx(stringutil.GetSelfFuncName() + "-" + idutil.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID) - log.ZDebug( - ctx, - "UserMsgsDestruct", - "conversationID", - conversation.ConversationID, - "ownerUserID", - conversation.OwnerUserID, - "msgDestructTime", - conversation.MsgDestructTime, - "lastMsgDestructTime", - conversation.LatestMsgDestructTime, - ) - now := time.Now() - seqs, err := c.msgDatabase.UserMsgsDestruct(ctx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) - if err != nil { - log.ZError(ctx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) - continue - } - if len(seqs) > 0 { - if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]any{"latest_msg_destruct_time": now}); err != nil { - log.ZError(ctx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) - continue - } - c.msgNotificationSender.UserDeleteMsgsNotification(ctx, conversation.OwnerUserID, conversation.ConversationID, seqs) - } - } - } -} diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 82ce95eda9..d52ed65096 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -16,8 +16,11 @@ package tools import ( "context" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/db/redisutil" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/tools/mcontext" "os" "os/signal" "syscall" @@ -25,7 +28,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/redis/go-redis/v9" "github.com/robfig/cron/v3" ) @@ -39,82 +41,43 @@ type CronTaskConfig struct { } func Start(ctx context.Context, config *CronTaskConfig) error { - - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", - config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.MsgDestructTime) - - msgTool, err := InitMsgTool(ctx, config) - if err != nil { - return err - } - - msgTool.convertTools() - - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) - if err != nil { - return err - } - - // register cron tasks - var crontab = cron.New() - - _, err = crontab.AddFunc(config.CronTask.ChatRecordsClearTime, - cronWrapFunc(config, rdb, "cron_clear_msg_and_fix_seq", msgTool.AllConversationClearMsgAndFixSeq)) - if err != nil { - return errs.Wrap(err) - } - - _, err = crontab.AddFunc(config.CronTask.MsgDestructTime, - cronWrapFunc(config, rdb, "cron_conversations_destruct_msgs", msgTool.ConversationsDestructMsgs)) - if err != nil { - return errs.WrapMsg(err, "cron_conversations_destruct_msgs") - } - - // start crontab - crontab.Start() - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - <-sigs - - // stop crontab, Wait for the running task to exit. - cronCtx := crontab.Stop() - - select { - case <-cronCtx.Done(): - // graceful exit - - case <-time.After(15 * time.Second): - // forced exit on timeout + log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.RetainChatRecords) + if config.CronTask.RetainChatRecords < 1 { + return errs.New("msg destruct time must be greater than 1").Wrap() } - - return nil -} - -// netlock redis lock. -func netlock(rdb redis.UniversalClient, key string, ttl time.Duration) bool { - value := "used" - ok, err := rdb.SetNX(context.Background(), key, value, ttl).Result() // nolint + client, err := kdisc.NewDiscoveryRegister(&config.ZookeeperConfig, &config.Share) if err != nil { - // when err is about redis server, return true. - return false + return errs.WrapMsg(err, "failed to register discovery service") } - - return ok -} - -func cronWrapFunc(config *CronTaskConfig, rdb redis.UniversalClient, key string, fn func()) func() { - enableCronLocker := config.CronTask.EnableCronLocker - return func() { - // if don't enable cron-locker, call fn directly. - if !enableCronLocker { - fn() - return + cli := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + ctx, exitBy := context.WithCancelCause(context.Background()) + ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + select { + case <-ctx.Done(): + case s := <-sigs: + exitBy(fmt.Errorf("exit signal %s", s)) } - - // when acquire redis lock, call fn(). - if netlock(rdb, key, 5*time.Second) { - fn() + }() + crontab := cron.New() + clearFunc := func() { + now := time.Now() + deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + if err := cli.ClearMsg(ctx, deltime.UnixMilli()); err != nil { + log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) + return } + log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { + return errs.Wrap(err) } + log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) + crontab.Start() + <-ctx.Done() + return context.Cause(ctx) } diff --git a/internal/tools/cron_task_test.go b/internal/tools/cron_task_test.go deleted file mode 100644 index 0bea8a4361..0000000000 --- a/internal/tools/cron_task_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "testing" - "time" - - "github.com/redis/go-redis/v9" - "github.com/stretchr/testify/assert" -) - -func TestDisLock(t *testing.T) { - rdb := redis.NewClient(&redis.Options{}) - defer rdb.Close() - - assert.Equal(t, true, netlock(rdb, "cron-1", 1*time.Second)) - - // if exists, get false - assert.Equal(t, false, netlock(rdb, "cron-1", 1*time.Second)) - - time.Sleep(2 * time.Second) - - // wait for key on timeout, get true - assert.Equal(t, true, netlock(rdb, "cron-1", 2*time.Second)) - - // set different key - assert.Equal(t, true, netlock(rdb, "cron-2", 2*time.Second)) -} - -//func TestCronWrapFunc(t *testing.T) { -// rdb := redis.NewClient(&redis.Options{}) -// defer rdb.Close() -// -// once := sync.Once{} -// done := make(chan struct{}, 1) -// cb := func() { -// once.Do(func() { -// close(done) -// }) -// } -// -// start := time.Now() -// key := fmt.Sprintf("cron-%v", rand.Int31()) -// crontab := cron.New(cron.WithSeconds()) -// crontab.AddFunc("*/1 * * * * *", cronWrapFunc(config.NewGlobalConfig(), rdb, key, cb)) -// crontab.Start() -// <-done -// -// dur := time.Since(start) -// assert.LessOrEqual(t, dur.Seconds(), float64(2*time.Second)) -// crontab.Stop() -//} -// -//func TestCronWrapFuncWithNetlock(t *testing.T) { -// conf, err := initCfg() -// if err != nil { -// panic(err) -// } -// conf.EnableCronLocker = true -// rdb := redis.NewClient(&redis.Options{}) -// defer rdb.Close() -// -// done := make(chan string, 10) -// -// crontab := cron.New(cron.WithSeconds()) -// -// key := fmt.Sprintf("cron-%v", rand.Int31()) -// crontab.AddFunc("*/1 * * * * *", cronWrapFunc(conf, rdb, key, func() { -// done <- "host1" -// })) -// crontab.AddFunc("*/1 * * * * *", cronWrapFunc(conf, rdb, key, func() { -// done <- "host2" -// })) -// crontab.Start() -// -// time.Sleep(12 * time.Second) -// // the ttl of netlock is 5s, so expected value is 2. -// assert.Equal(t, len(done), 2) -// -// crontab.Stop() -//} -// -//func initCfg() (*config.GlobalConfig, error) { -// const ( -// defaultCfgPath = "../../../../../config/config.yaml" -// ) -// -// cfgPath := flag.String("c", defaultCfgPath, "Path to the configuration file") -// data, err := os.ReadFile(*cfgPath) -// if err != nil { -// return nil, errs.WrapMsg(err, "ReadFile unmarshal failed") -// } -// -// conf := config.NewGlobalConfig() -// err = yaml.Unmarshal(data, &conf) -// if err != nil { -// return nil, errs.WrapMsg(err, "InitConfig unmarshal failed") -// } -// return conf, nil -//} diff --git a/internal/tools/msg.go b/internal/tools/msg.go deleted file mode 100644 index 42af1efdc2..0000000000 --- a/internal/tools/msg.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "context" - "fmt" - "github.com/openimsdk/open-im-server/v3/internal/rpc/msg" - "math" - "math/rand" - - "github.com/openimsdk/open-im-server/v3/pkg/common/db/controller" - "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/stringutil" - "github.com/redis/go-redis/v9" -) - -type MsgTool struct { - msgDatabase controller.CommonMsgDatabase - conversationDatabase controller.ConversationDatabase - userDatabase controller.UserDatabase - groupDatabase controller.GroupDatabase - msgNotificationSender *msg.MsgNotificationSender - config *CronTaskConfig -} - -func NewMsgTool(msgDatabase controller.CommonMsgDatabase, userDatabase controller.UserDatabase, - groupDatabase controller.GroupDatabase, conversationDatabase controller.ConversationDatabase, - msgNotificationSender *msg.MsgNotificationSender, config *CronTaskConfig, -) *MsgTool { - return &MsgTool{ - msgDatabase: msgDatabase, - userDatabase: userDatabase, - groupDatabase: groupDatabase, - conversationDatabase: conversationDatabase, - msgNotificationSender: msgNotificationSender, - config: config, - } -} - -func InitMsgTool(ctx context.Context, config *CronTaskConfig) (*MsgTool, error) { - ch := make(chan int) - <-ch - //mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) - //if err != nil { - // return nil, err - //} - //rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) - //if err != nil { - // return nil, err - //} - //discov, err := kdisc.NewDiscoveryRegister(&config.ZookeeperConfig, &config.Share) - //if err != nil { - // return nil, err - //} - //discov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - //userDB, err := mgo.NewUserMongo(mgocli.GetDB()) - //if err != nil { - // return nil, err - //} - ////msgDatabase, err := controller.InitCommonMsgDatabase(rdb, mgocli.GetDB(), config) - //if err != nil { - // return nil, err - //} - //userMongoDB := mgo.NewUserMongoDriver(mgocli.GetDB()) - //userDatabase := controller.NewUserDatabase( - // userDB, - // cache.NewUserCacheRedis(rdb, userDB, cache.GetDefaultOpt()), - // mgocli.GetTx(), - // userMongoDB, - //) - //groupDB, err := mgo.NewGroupMongo(mgocli.GetDB()) - //if err != nil { - // return nil, err - //} - //groupMemberDB, err := mgo.NewGroupMember(mgocli.GetDB()) - //if err != nil { - // return nil, err - //} - //groupRequestDB, err := mgo.NewGroupRequestMgo(mgocli.GetDB()) - //if err != nil { - // return nil, err - //} - //conversationDB, err := mgo.NewConversationMongo(mgocli.GetDB()) - //if err != nil { - // return nil, err - //} - //groupDatabase := controller.NewGroupDatabase(rdb, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), nil) - //conversationDatabase := controller.NewConversationDatabase( - // conversationDB, - // cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), conversationDB), - // mgocli.GetTx(), - //) - //msgRpcClient := rpcclient.NewMessageRpcClient(discov, config.Share.RpcRegisterName.Msg) - //msgNotificationSender := notification.NewMsgNotificationSender(config, rpcclient.WithRpcClient(&msgRpcClient)) - //msgTool := NewMsgTool(msgDatabase, userDatabase, groupDatabase, conversationDatabase, msgNotificationSender, config) - //return msgTool, nil - return nil, nil -} - -// func (c *MsgTool) AllConversationClearMsgAndFixSeq() { -// ctx := mcontext.NewCtx(utils.GetSelfFuncName()) -// log.ZInfo(ctx, "============================ start del cron task ============================") -// conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx) -// if err != nil { -// log.ZError(ctx, "GetAllConversationIDs failed", err) -// return -// } -// for _, conversationID := range conversationIDs { -// conversationIDs = append(conversationIDs, utils.GetNotificationConversationIDByConversationID(conversationID)) -// } -// c.ClearConversationsMsg(ctx, conversationIDs) -// log.ZInfo(ctx, "============================ start del cron finished ============================") -//} - -func (c *MsgTool) AllConversationClearMsgAndFixSeq() { - ctx := mcontext.NewCtx(stringutil.GetSelfFuncName()) - log.ZInfo(ctx, "============================ start del cron task ============================") - num, err := c.conversationDatabase.GetAllConversationIDsNumber(ctx) - if err != nil { - log.ZError(ctx, "GetAllConversationIDsNumber failed", err) - return - } - const batchNum = 50 - log.ZDebug(ctx, "GetAllConversationIDsNumber", "num", num) - if num == 0 { - return - } - count := int(num/batchNum + num/batchNum/2) - if count < 1 { - count = 1 - } - maxPage := 1 + num/batchNum - if num%batchNum != 0 { - maxPage++ - } - for i := 0; i < count; i++ { - pageNumber := rand.Int63() % maxPage - pagination := &sdkws.RequestPagination{ - PageNumber: int32(pageNumber), - ShowNumber: batchNum, - } - conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) - if err != nil { - log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) - continue - } - log.ZDebug(ctx, "PageConversationIDs failed", "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) - if len(conversationIDs) == 0 { - continue - } - c.ClearConversationsMsg(ctx, conversationIDs) - } - log.ZInfo(ctx, "============================ start del cron finished ============================") -} - -func (c *MsgTool) ClearConversationsMsg(ctx context.Context, conversationIDs []string) { - for _, conversationID := range conversationIDs { - if err := c.msgDatabase.DeleteConversationMsgsAndSetMinSeq(ctx, conversationID, int64(c.config.CronTask.RetainChatRecords*24*60*60)); err != nil { - log.ZError(ctx, "DeleteUserSuperGroupMsgsAndSetMinSeq failed", err, "conversationID", - conversationID, "DBRetainChatRecords", c.config.CronTask.RetainChatRecords) - } - if err := c.checkMaxSeq(ctx, conversationID); err != nil { - log.ZError(ctx, "fixSeq failed", err, "conversationID", conversationID) - } - } -} - -func (c *MsgTool) checkMaxSeqWithMongo(ctx context.Context, conversationID string, maxSeqCache int64) error { - minSeqMongo, maxSeqMongo, err := c.msgDatabase.GetMongoMaxAndMinSeq(ctx, conversationID) - if err != nil { - return err - } - if math.Abs(float64(maxSeqMongo-maxSeqCache)) > 10 { - err = fmt.Errorf("cache max seq and mongo max seq is diff > 10, maxSeqMongo:%d,minSeqMongo:%d,maxSeqCache:%d,conversationID:%s", maxSeqMongo, minSeqMongo, maxSeqCache, conversationID) - return errs.Wrap(err) - } - return nil -} - -func (c *MsgTool) checkMaxSeq(ctx context.Context, conversationID string) error { - maxSeq, err := c.msgDatabase.GetMaxSeq(ctx, conversationID) - if err != nil { - if errs.Unwrap(err) == redis.Nil { - return nil - } - return err - } - if err := c.checkMaxSeqWithMongo(ctx, conversationID, maxSeq); err != nil { - return err - } - return nil -} - -func (c *MsgTool) FixAllSeq(ctx context.Context) error { - conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx) - if err != nil { - return err - } - for _, conversationID := range conversationIDs { - conversationIDs = append(conversationIDs, conversationutil.GetNotificationConversationIDByConversationID(conversationID)) - } - for _, conversationID := range conversationIDs { - if err := c.checkMaxSeq(ctx, conversationID); err != nil { - log.ZWarn(ctx, "fixSeq failed", err, "conversationID", conversationID) - } - } - fmt.Println("fix all seq finished") - return nil -} diff --git a/internal/tools/msg_doc_convert.go b/internal/tools/msg_doc_convert.go deleted file mode 100644 index e7625ce215..0000000000 --- a/internal/tools/msg_doc_convert.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" -) - -func (c *MsgTool) convertTools() { - ctx := mcontext.NewCtx("convert") - conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx) - if err != nil { - log.ZError(ctx, "get all conversation ids failed", err) - return - } - for _, conversationID := range conversationIDs { - conversationIDs = append(conversationIDs, msgprocessor.GetNotificationConversationIDByConversationID(conversationID)) - } - _, userIDs, err := c.userDatabase.GetAllUserID(ctx, nil) - if err != nil { - log.ZError(ctx, "get all user ids failed", err) - return - } - log.ZDebug(ctx, "all userIDs", "len userIDs", len(userIDs)) - for _, userID := range userIDs { - conversationIDs = append(conversationIDs, msgprocessor.GetConversationIDBySessionType(constant.SingleChatType, userID, userID)) - conversationIDs = append(conversationIDs, msgprocessor.GetNotificationConversationID(constant.SingleChatType, userID, userID)) - } - log.ZDebug(ctx, "all conversationIDs", "len userIDs", len(conversationIDs)) - c.msgDatabase.ConvertMsgsDocLen(ctx, conversationIDs) -} diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index a96d6de201..f1b377bad7 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -46,7 +46,6 @@ func IsAppManagerUid(ctx context.Context, imAdminUserID []string) bool { } func CheckAdmin(ctx context.Context, imAdminUserID []string) error { - if datautil.Contain(mcontext.GetOpUserID(ctx), imAdminUserID...) { return nil } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index df2639cab3..caf31036cd 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -107,9 +107,7 @@ type API struct { type CronTask struct { ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` - MsgDestructTime string `mapstructure:"msgDestructTime"` RetainChatRecords int `mapstructure:"retainChatRecords"` - EnableCronLocker bool `yaml:"enableCronLocker"` } type OfflinePushConfig struct { diff --git a/pkg/common/db/controller/msg.go b/pkg/common/db/controller/msg.go index 130e35d20e..c0a013c175 100644 --- a/pkg/common/db/controller/msg.go +++ b/pkg/common/db/controller/msg.go @@ -17,6 +17,8 @@ package controller import ( "context" "encoding/json" + "errors" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -102,6 +104,10 @@ type CommonMsgDatabase interface { RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*relation.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) + + // clear msg + GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*relation.MsgDocModel, error) + DeleteDocMsgBefore(ctx context.Context, ts int64, doc *relation.MsgDocModel) ([]int, error) } func NewCommonMsgDatabase(msgDocModel relation.MsgDocModelInterface, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -1047,3 +1053,72 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) { db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } + +func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*relation.MsgDocModel, error) { + return db.msgDocDatabase.GetBeforeMsg(ctx, ts, limit) +} + +func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *relation.MsgDocModel) ([]int, error) { + var notNull int + index := make([]int, 0, len(doc.Msg)) + for i, message := range doc.Msg { + if message.Msg != nil { + notNull++ + if message.Msg.SendTime < ts { + index = append(index, i) + } + } + } + if len(index) == 0 { + return index, nil + } + maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq + conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] + if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil { + return index, err + } + if len(index) == notNull { + return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) + } else { + return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) + } +} + +//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) { +// var ( +// docNum int +// msgNum int +// start = time.Now() +// ) +// for { +// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100) +// if err != nil { +// return err +// } +// if len(msgs) == 0 { +// return nil +// } +// for _, msg := range msgs { +// num, err := db.deleteOneMsg(ctx, ts, msg) +// if err != nil { +// return err +// } +// docNum++ +// msgNum += num +// } +// } +//} + +func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { + dbSeq, err := db.seq.GetMinSeq(ctx, conversationID) + if err != nil { + if errors.Is(errs.Unwrap(err), redis.Nil) { + return nil + } + return err + } + if dbSeq >= seq { + return nil + } + return db.seq.SetMinSeq(ctx, conversationID, seq) +} diff --git a/pkg/common/db/mgo/conversation.go b/pkg/common/db/mgo/conversation.go index d10bda39b1..5292cb60cf 100644 --- a/pkg/common/db/mgo/conversation.go +++ b/pkg/common/db/mgo/conversation.go @@ -56,7 +56,16 @@ func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err er } func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) { - res, err := mongoutil.UpdateMany(ctx, c.coll, bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": conversationID}, bson.M{"$set": args}) + if len(args) == 0 { + return 0, nil + } + filter := bson.M{ + "conversation_id": conversationID, + } + if len(userIDs) > 0 { + filter["owner_user_id"] = bson.M{"$in": userIDs} + } + res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) if err != nil { return 0, err } diff --git a/pkg/common/db/mgo/msg.go b/pkg/common/db/mgo/msg.go index 40ee5e4235..6fe24536bd 100644 --- a/pkg/common/db/mgo/msg.go +++ b/pkg/common/db/mgo/msg.go @@ -896,3 +896,89 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } } + +func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*relation.MsgDocModel, error) { + return mongoutil.Aggregate[*relation.MsgDocModel](ctx, m.coll, []bson.M{ + { + "$match": bson.M{ + "msgs.msg.send_time": bson.M{ + "$lt": ts, + }, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 1, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + { + "$limit": limit, + }, + }) +} + +func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int) error { + if len(index) == 0 { + return nil + } + model := &relation.MsgInfoModel{DelList: []string{}} + set := make(map[string]any) + for i := range index { + set[fmt.Sprintf("msgs.%d", i)] = model + } + return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) +} + +//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) { +// ts := t.UnixMilli() +// var count int64 +// for { +// msgs, err := m.GetBeforeMsg(ctx, ts, 100) +// if err != nil { +// return count, err +// } +// if len(msgs) == 0 { +// return count, nil +// } +// for _, msg := range msgs { +// num, err := m.deleteOneMsg(ctx, ts, msg) +// count += num +// if err != nil { +// return count, err +// } +// } +// } +//} + +func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { + return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) +} + +//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) { +// var notNull int +// index := make([]int, 0, len(doc.Msg)) +// for i, message := range doc.Msg { +// if message.Msg != nil { +// notNull++ +// if message.Msg.SendTime < ts { +// index = append(index, i) +// } +// } +// } +// if len(index) == 0 { +// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID) +// } +// if len(index) == notNull { +// if err := m.DeleteDoc(ctx, doc.DocID); err != nil { +// return 0, err +// } +// } else { +// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil { +// return 0, err +// } +// } +// return int64(len(index)), nil +//} diff --git a/pkg/common/db/mgo/msg_del.go b/pkg/common/db/mgo/msg_del.go new file mode 100644 index 0000000000..cee5fa065b --- /dev/null +++ b/pkg/common/db/mgo/msg_del.go @@ -0,0 +1,71 @@ +package mgo + +import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "go.mongodb.org/mongo-driver/mongo" +) + +func NewMsgDelMongo(db *mongo.Database) *MsgDelMgo { + return &MsgDelMgo{coll: db.Collection(new(relation.MsgDocModel).TableName())} +} + +type MsgDelMgo struct { + coll *mongo.Collection + model relation.MsgDocModel +} + +//func (m *MsgDelMgo) getEmptyMsg(ctx context.Context, limit int) ([]string, error) { +// return mongoutil.Aggregate[string](ctx, m.coll, []bson.M{ +// { +// "$match": bson.M{ +// "msgs": bson.M{ +// "$exists": true, +// }, +// }, +// }, +// { +// "$project": bson.M{ +// "_id": 0, +// "doc_id": 1, +// "all_null_msgs": bson.M{ +// "$not": []bson.M{ +// { +// "$anyElementTrue": bson.M{ +// "$map": bson.M{ +// "input": "$msgs", +// "as": "item", +// "in": "$$item.msg", +// }, +// }, +// }, +// }, +// }, +// }, +// }, +// { +// "$project": bson.M{ +// "doc_id": 1, +// }, +// }, +// { +// "$limit": limit, +// }, +// }) +//} +// +//func (m *MsgDelMgo) deleteEmptyMsgs(ctx context.Context) error { +// for { +// docIDs, err := m.getEmptyMsg(ctx, 100) +// if err != nil { +// return err +// } +// if len(docIDs) == 0 { +// return nil +// } +// for _, docID := range docIDs { +// if err := m.deleteEmptyMsg(ctx, docID); err != nil { +// return err +// } +// } +// } +//} diff --git a/pkg/common/db/mgo/msg_test.go b/pkg/common/db/mgo/msg_test.go new file mode 100644 index 0000000000..15026d69c6 --- /dev/null +++ b/pkg/common/db/mgo/msg_test.go @@ -0,0 +1,46 @@ +package mgo + +import ( + "strings" + "testing" +) + +func TestName(t *testing.T) { + //conf := config.Mongo{ + // Address: []string{"localhost:37017"}, + // Username: "openIM", + // Password: "openIM123", + // Database: "demo", + //} + //conf.URI = `mongodb://openIM:openIM123@localhost:37017/demo?maxPoolSize=100&authSource=admin` + //cli, err := mongoutil.NewMongoDB(context.Background(), conf.Build()) + //if err != nil { + // panic(err) + //} + //msg, _ := NewMsgMongo(cli.GetDB()) + //count, err := msg.ClearMsg(context.Background(), time.Now().Add(-time.Hour*24*61)) + //if err != nil { + // t.Log("error", err) + // return + //} + //t.Log("count", count) + + s := `si_5300327160_9129042887:0123` + + t.Log(s[:strings.LastIndex(s, ":")]) + +} + +func TestName2(t *testing.T) { + m := map[string]string{ + "1": "1", + "2": "2", + } + t.Log(m) + clear(m) + t.Log(m) + a := []string{"1", "2"} + t.Log(a) + clear(a) + t.Log(a) +} diff --git a/pkg/common/db/table/relation/msg.go b/pkg/common/db/table/relation/msg.go index 14b8dbb30e..41a6ede974 100644 --- a/pkg/common/db/table/relation/msg.go +++ b/pkg/common/db/table/relation/msg.go @@ -116,6 +116,12 @@ type MsgDocModelInterface interface { RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) + + DeleteDoc(ctx context.Context, docID string) error + DeleteMsgByIndex(ctx context.Context, docID string, index []int) error + GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*MsgDocModel, error) + + //ClearMsg(ctx context.Context, t time.Time) (int64, error) } func (MsgDocModel) TableName() string { diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go index 6eb6c94612..e078f432b3 100644 --- a/pkg/rpcclient/conversation.go +++ b/pkg/rpcclient/conversation.go @@ -82,6 +82,11 @@ func (c *ConversationRpcClient) SetConversations(ctx context.Context, userIDs [] return err } +func (c *ConversationRpcClient) UpdateConversations(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { + _, err := c.Client.UpdateConversation(ctx, conversation) + return err +} + func (c *ConversationRpcClient) GetConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { resp, err := c.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID}) if err != nil { diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 36daf9f66b..f660c74dd7 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -212,6 +212,11 @@ func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversati return resp.MaxSeq, nil } +func (m *MessageRpcClient) ClearMsg(ctx context.Context, ts int64) error { + _, err := m.Client.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: ts}) + return err +} + type NotificationSender struct { contentTypeConf map[int32]config.NotificationConfig sessionTypeConf map[int32]int32 From 2f13149555c1cd10546d778fee17d4a6854bd5ab Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 25 Apr 2024 17:02:00 +0800 Subject: [PATCH 008/147] feat: cron task --- go.mod | 7 +------ go.sum | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 58354b107d..6b23243b66 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.64 + github.com/openimsdk/protocol v0.0.65 github.com/openimsdk/tools v0.0.49-alpha.2 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -174,8 +174,3 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - - -replace ( - github.com/openimsdk/protocol => /Users/chao/Desktop/withchao/protocol -) \ No newline at end of file diff --git a/go.sum b/go.sum index 5dc0cc524c..aa851c74b8 100644 --- a/go.sum +++ b/go.sum @@ -22,10 +22,13 @@ github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mo github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -43,6 +46,7 @@ github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5P github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,12 +70,15 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -90,6 +97,7 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -137,11 +145,13 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= @@ -194,6 +204,7 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -209,7 +220,9 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= @@ -268,8 +281,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.9 h1:ouf25ygN2PMQ68Gfgns/EQRPiLPnp+77SIr68GfE+n4= github.com/openimsdk/gomake v0.0.9/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.64 h1:OrjSs4CgKN9VLvJvrAsc37O7Ru0E0VllXZQSmG/ab7U= -github.com/openimsdk/protocol v0.0.64/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= +github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.2 h1:8IfV6o2ySU7C54sh/MG7ctEp1h3lSNe03OCUDWSk5Ws= github.com/openimsdk/tools v0.0.49-alpha.2/go.mod h1:P4oGP1Pd+d4ctbLD5U/XQTgl8yu8Hd3skx640Fr69ko= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -282,6 +295,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -298,6 +312,7 @@ github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -376,6 +391,7 @@ go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFu go.opentelemetry.io/otel/metric v1.23.0 h1:pazkx7ss4LFVVYSxYew7L5I6qvLXHA0Ap2pwV+9Cnpo= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.23.0 h1:37Ik5Ib7xfYVb4V1UtnT97T1jI+AoIYkJyPkuL4iJgI= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -383,6 +399,7 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -489,6 +506,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.165.0 h1:zd5d4JIIIaYYsfVy1HzoXYZ9rWCSBxxAglbczzo7Bgc= google.golang.org/api v0.165.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -526,6 +544,7 @@ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGm google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= From 2e439508bcb822f878e83b36bb8c39218c9d01e3 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 25 Apr 2024 17:09:24 +0800 Subject: [PATCH 009/147] feat: cron task --- pkg/common/cmd/msg_utils.go | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/pkg/common/cmd/msg_utils.go b/pkg/common/cmd/msg_utils.go index fc5eaceae8..a0a9b04101 100644 --- a/pkg/common/cmd/msg_utils.go +++ b/pkg/common/cmd/msg_utils.go @@ -15,16 +15,11 @@ package cmd import ( - "context" - - "github.com/openimsdk/open-im-server/v3/internal/tools" - "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) type MsgUtilsCmd struct { cobra.Command - MsgTool *tools.MsgTool } func (m *MsgUtilsCmd) AddUserIDFlag() { @@ -146,27 +141,7 @@ func NewSeqCmd() *SeqCmd { func (s *SeqCmd) GetSeqCmd() *cobra.Command { s.Command.Run = func(cmdLines *cobra.Command, args []string) { - _, err := tools.InitMsgTool(context.Background(), nil) - if err != nil { - program.ExitWithError(err) - } - userID := s.getUserIDFlag(cmdLines) - superGroupID := s.getSuperGroupIDFlag(cmdLines) - // beginSeq := s.getBeginSeqFlag(cmdLines) - // limit := s.getLimitFlag(cmdLines) - if userID != "" { - // seq, err := msgTool.s(context.Background(), userID) - if err != nil { - panic(err) - } - // println(seq) - } else if superGroupID != "" { - // seq, err := msgTool.GetSuperGroupSeq(context.Background(), superGroupID) - if err != nil { - panic(err) - } - // println(seq) - } + } return &s.Command } From 7a29a853e9e50f87925ff96d6bcbdb4317d47146 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 26 Apr 2024 15:32:20 +0800 Subject: [PATCH 010/147] feat: cron task --- internal/tools/cron_task.go | 17 +++++++---- pkg/common/cmd/cron_task.go | 3 -- pkg/common/db/mgo/msg_test.go | 56 ++++++++++++++--------------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index d52ed65096..7161f55fc1 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -19,8 +19,11 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "os" "os/signal" "syscall" @@ -33,11 +36,8 @@ import ( type CronTaskConfig struct { CronTask config.CronTask - RedisConfig config.Redis - MongodbConfig config.Mongo ZookeeperConfig config.ZooKeeper Share config.Share - KafkaConfig config.Kafka } func Start(ctx context.Context, config *CronTaskConfig) error { @@ -49,9 +49,14 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } - cli := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx, exitBy := context.WithCancelCause(context.Background()) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) + conn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) + if err != nil { + return err + } + cli := msg.NewMsgClient(conn) go func() { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGTERM) @@ -67,7 +72,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - if err := cli.ClearMsg(ctx, deltime.UnixMilli()); err != nil { + if _, err := cli.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index 0e94cf52ce..be26f5af36 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -34,11 +34,8 @@ func NewCronTaskCmd() *CronTaskCmd { ret := &CronTaskCmd{cronTaskConfig: &cronTaskConfig} ret.configMap = map[string]any{ OpenIMCronTaskCfgFileName: &cronTaskConfig.CronTask, - RedisConfigFileName: &cronTaskConfig.RedisConfig, - MongodbConfigFileName: &cronTaskConfig.MongodbConfig, ZookeeperConfigFileName: &cronTaskConfig.ZookeeperConfig, ShareFileName: &cronTaskConfig.Share, - KafkaConfigFileName: &cronTaskConfig.KafkaConfig, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) diff --git a/pkg/common/db/mgo/msg_test.go b/pkg/common/db/mgo/msg_test.go index 15026d69c6..0a92b7d4ad 100644 --- a/pkg/common/db/mgo/msg_test.go +++ b/pkg/common/db/mgo/msg_test.go @@ -1,46 +1,34 @@ package mgo import ( - "strings" + "context" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "testing" + "time" ) func TestName(t *testing.T) { - //conf := config.Mongo{ - // Address: []string{"localhost:37017"}, - // Username: "openIM", - // Password: "openIM123", - // Database: "demo", - //} - //conf.URI = `mongodb://openIM:openIM123@localhost:37017/demo?maxPoolSize=100&authSource=admin` - //cli, err := mongoutil.NewMongoDB(context.Background(), conf.Build()) - //if err != nil { - // panic(err) - //} - //msg, _ := NewMsgMongo(cli.GetDB()) - //count, err := msg.ClearMsg(context.Background(), time.Now().Add(-time.Hour*24*61)) - //if err != nil { - // t.Log("error", err) - // return - //} - //t.Log("count", count) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + ctx = mcontext.SetOpUserID(ctx, "imAdmin") + ctx = mcontext.SetOperationID(ctx, "test123456") - s := `si_5300327160_9129042887:0123` - - t.Log(s[:strings.LastIndex(s, ":")]) + conn, err := grpc.DialContext(ctx, "172.16.8.48:10130", grpc.WithTransportCredentials(insecure.NewCredentials()), mw.GrpcClient()) + if err != nil { + panic(err) + } + defer conn.Close() + cli := msg.NewMsgClient(conn) + var ts int64 -} + ts = time.Now().UnixMilli() -func TestName2(t *testing.T) { - m := map[string]string{ - "1": "1", - "2": "2", + if _, err := cli.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: ts}); err != nil { + panic(err) } - t.Log(m) - clear(m) - t.Log(m) - a := []string{"1", "2"} - t.Log(a) - clear(a) - t.Log(a) + t.Log("success!") } From 3920c0673c6acd027cba6a3833d27742aee3f9ba Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 26 Apr 2024 15:33:03 +0800 Subject: [PATCH 011/147] feat: cron task --- pkg/common/db/mgo/msg_del.go | 71 ----------------------------------- pkg/common/db/mgo/msg_test.go | 34 ----------------- 2 files changed, 105 deletions(-) delete mode 100644 pkg/common/db/mgo/msg_del.go delete mode 100644 pkg/common/db/mgo/msg_test.go diff --git a/pkg/common/db/mgo/msg_del.go b/pkg/common/db/mgo/msg_del.go deleted file mode 100644 index cee5fa065b..0000000000 --- a/pkg/common/db/mgo/msg_del.go +++ /dev/null @@ -1,71 +0,0 @@ -package mgo - -import ( - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" - "go.mongodb.org/mongo-driver/mongo" -) - -func NewMsgDelMongo(db *mongo.Database) *MsgDelMgo { - return &MsgDelMgo{coll: db.Collection(new(relation.MsgDocModel).TableName())} -} - -type MsgDelMgo struct { - coll *mongo.Collection - model relation.MsgDocModel -} - -//func (m *MsgDelMgo) getEmptyMsg(ctx context.Context, limit int) ([]string, error) { -// return mongoutil.Aggregate[string](ctx, m.coll, []bson.M{ -// { -// "$match": bson.M{ -// "msgs": bson.M{ -// "$exists": true, -// }, -// }, -// }, -// { -// "$project": bson.M{ -// "_id": 0, -// "doc_id": 1, -// "all_null_msgs": bson.M{ -// "$not": []bson.M{ -// { -// "$anyElementTrue": bson.M{ -// "$map": bson.M{ -// "input": "$msgs", -// "as": "item", -// "in": "$$item.msg", -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// { -// "$project": bson.M{ -// "doc_id": 1, -// }, -// }, -// { -// "$limit": limit, -// }, -// }) -//} -// -//func (m *MsgDelMgo) deleteEmptyMsgs(ctx context.Context) error { -// for { -// docIDs, err := m.getEmptyMsg(ctx, 100) -// if err != nil { -// return err -// } -// if len(docIDs) == 0 { -// return nil -// } -// for _, docID := range docIDs { -// if err := m.deleteEmptyMsg(ctx, docID); err != nil { -// return err -// } -// } -// } -//} diff --git a/pkg/common/db/mgo/msg_test.go b/pkg/common/db/mgo/msg_test.go deleted file mode 100644 index 0a92b7d4ad..0000000000 --- a/pkg/common/db/mgo/msg_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package mgo - -import ( - "context" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/mw" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "testing" - "time" -) - -func TestName(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - ctx = mcontext.SetOpUserID(ctx, "imAdmin") - ctx = mcontext.SetOperationID(ctx, "test123456") - - conn, err := grpc.DialContext(ctx, "172.16.8.48:10130", grpc.WithTransportCredentials(insecure.NewCredentials()), mw.GrpcClient()) - if err != nil { - panic(err) - } - defer conn.Close() - cli := msg.NewMsgClient(conn) - var ts int64 - - ts = time.Now().UnixMilli() - - if _, err := cli.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: ts}); err != nil { - panic(err) - } - t.Log("success!") -} From 480ccc70b0512219f98913ac02550f3a77586254 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Sun, 28 Apr 2024 18:41:36 +0800 Subject: [PATCH 012/147] fix: minio config url recognition error --- pkg/common/config/config.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index caf31036cd..24d04d8ccc 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,14 +15,13 @@ package config import ( - "fmt" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "net" + "strings" "time" ) @@ -473,25 +472,23 @@ func (k *Kafka) Build() *kafka.Config { }, } } + func (m *Minio) Build() *minio.Config { - conf := minio.Config{ + formatEndpoint := func(address string) string { + if strings.HasPrefix(address, "http://") || strings.HasPrefix(address, "https://") { + return address + } + return "http://" + address + } + return &minio.Config{ Bucket: m.Bucket, AccessKeyID: m.AccessKeyID, SecretAccessKey: m.SecretAccessKey, SessionToken: m.SessionToken, PublicRead: m.PublicRead, + Endpoint: formatEndpoint(m.InternalAddress), + SignEndpoint: formatEndpoint(m.ExternalAddress), } - if _, _, err := net.SplitHostPort(m.InternalAddress); err == nil { - conf.Endpoint = fmt.Sprintf("http://%s", m.InternalAddress) - } else { - conf.Endpoint = m.InternalAddress - } - if _, _, err := net.SplitHostPort(m.ExternalAddress); err == nil { - conf.SignEndpoint = fmt.Sprintf("http://%s", m.ExternalAddress) - } else { - conf.SignEndpoint = m.ExternalAddress - } - return &conf } func (c *Cos) Build() *cos.Config { return &cos.Config{ From 5528e8e44d0ed2e283f2f03bb092ef02b6e46dbc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 May 2024 10:54:25 +0800 Subject: [PATCH 013/147] new mongo --- pkg/common/listdemo/common.go | 182 ++++++++++++++++++++++++++++ pkg/common/listdemo/friend_model.go | 5 + pkg/common/listdemo/friend_table.go | 86 +++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 pkg/common/listdemo/common.go create mode 100644 pkg/common/listdemo/friend_model.go create mode 100644 pkg/common/listdemo/friend_table.go diff --git a/pkg/common/listdemo/common.go b/pkg/common/listdemo/common.go new file mode 100644 index 0000000000..2d99cd8c9f --- /dev/null +++ b/pkg/common/listdemo/common.go @@ -0,0 +1,182 @@ +package listdemo + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + ErrListNotFound = errors.New("list not found") + ErrElemExist = errors.New("elem exist") + ErrNotFound = mongo.ErrNoDocuments +) + +type ListDoc interface { + IDName() string // 外层业务id字段名字 user_id + ElemsName() string // 外层列表名字 friends + VersionName() string // 外层版本号 version + DeleteVersion() string // 删除版本号 + BuildDoc(lid any, e Elem) any // 返回一个组装的doc文档 +} + +type Elem interface { + IDName() string // 业务id名字 friend_user_id + IDValue() any // 业务id值 userID -> "100000000" + VersionName() string // 版本号 + DeletedName() string // 删除字段名字 + ToMap() map[string]any // 把结构体转换为map +} + +type List[D any, E Elem] struct { + coll *mongo.Collection + lf ListDoc +} + +func (l *List[D, E]) zeroE() E { + var t E + return t +} + +func (l *List[D, E]) FindElem(ctx context.Context, lid any, eid any) (E, error) { + res, err := l.FindElems(ctx, lid, []any{eid}) + if err != nil { + return l.zeroE(), err + } + if len(res) == 0 { + return l.zeroE(), ErrNotFound + } + return res[0], nil +} + +// FindElems 查询Elems +func (l *List[D, E]) FindElems(ctx context.Context, lid any, eids []any) ([]E, error) { + //pipeline := []bson.M{ + // { + // "$match": bson.M{ + // l.lf.IDName(): lid, + // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ + // "$in": eids, + // }, + // }, + // }, + // { + // "$unwind": "$" + l.lf.ElemsName(), + // }, + // { + // "$match": bson.M{ + // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ + // "$in": eids, + // }, + // }, + // }, + //} + panic("todo") +} + +func (l *List[D, E]) Find(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { + return nil, nil +} + +func (l *List[D, E]) Count(ctx context.Context, filter any, opts ...*options.CountOptions) (int64, error) { + return 0, nil +} + +func (l *List[D, E]) Update(ctx context.Context, lid any, eid any) (*mongo.UpdateResult, error) { + + return nil, nil +} + +func (l *List[D, E]) Delete(ctx context.Context, lid any, eids any) (*mongo.UpdateResult, error) { + + return nil, nil +} + +func (l *List[D, E]) Page(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []E, error) { + return 0, nil, nil +} + +func (l *List[D, E]) ElemIDs(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { + + return nil, nil +} + +// InsertElem 插入一个 +func (l *List[D, E]) InsertElem(ctx context.Context, lid any, e Elem) error { + if err := l.insertElem(ctx, lid, e); err == nil { + return nil + } else if !errors.Is(err, ErrListNotFound) { + return err + } + if _, err := l.coll.InsertOne(ctx, l.lf.BuildDoc(lid, e)); err == nil { + return nil + } else if mongo.IsDuplicateKeyError(err) { + return l.insertElem(ctx, lid, e) + } else { + return err + } +} + +func (l *List[D, E]) insertElem(ctx context.Context, lid any, e Elem) error { + data := e.ToMap() + data[e.VersionName()] = "$max_version" + filter := bson.M{ + l.lf.IDName(): lid, + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "found_elem": bson.M{ + "$in": bson.A{e.IDValue(), l.lf.ElemsName() + "." + e.IDName()}, + }, + }, + }, + { + "$set": bson.M{ + "max_version": bson.M{ + "$cond": bson.M{ + "if": "$found_elem", + "then": "$max_version", + "else": bson.M{"$add": bson.A{"max_version", 1}}, + }, + }, + }, + }, + { + "$set": bson.M{ + l.lf.ElemsName(): bson.M{ + "$cond": bson.M{ + "if": "$found_elem", + "then": "$" + l.lf.ElemsName(), + "else": bson.M{ + "$concatArrays": bson.A{ + "$" + l.lf.ElemsName(), + bson.A{ + data, + }, + }, + }, + }, + }, + }, + }, + { + "$unset": "found_elem", + }, + } + res, err := mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) + if err != nil { + return err + } + if res.MatchedCount == 0 { + return ErrListNotFound + } + if res.ModifiedCount == 0 { + return ErrElemExist + } + return nil +} diff --git a/pkg/common/listdemo/friend_model.go b/pkg/common/listdemo/friend_model.go new file mode 100644 index 0000000000..13507bfa3f --- /dev/null +++ b/pkg/common/listdemo/friend_model.go @@ -0,0 +1,5 @@ +package listdemo + +type friendModel struct { + db *List[*Friend, *FriendElem] +} diff --git a/pkg/common/listdemo/friend_table.go b/pkg/common/listdemo/friend_table.go new file mode 100644 index 0000000000..b1ad877a21 --- /dev/null +++ b/pkg/common/listdemo/friend_table.go @@ -0,0 +1,86 @@ +package listdemo + +import ( + "time" +) + +var ( + _ Elem = (*FriendElem)(nil) + _ ListDoc = (*Friend)(nil) +) + +type FriendElem struct { + FriendUserID string `bson:"friend_user_id"` + Nickname string `bson:"nickname"` + FaceURL string `bson:"face_url"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` + Version uint `bson:"version"` + DeleteTime *time.Time `bson:"delete_time"` +} + +func (f *FriendElem) IDName() string { + return "friend_user_id" +} + +func (f *FriendElem) IDValue() any { + return f.FriendUserID +} + +func (f *FriendElem) VersionName() string { + return "version" +} + +func (f *FriendElem) DeletedName() string { + return "delete_time" +} + +func (f *FriendElem) ToMap() map[string]any { + return map[string]any{ + "friend_user_id": f.FriendUserID, + "nickname": f.Nickname, + "face_url": f.FaceURL, + "remark": f.Remark, + "create_time": f.CreateTime, + "add_source": f.AddSource, + "operator_user_id": f.OperatorUserID, + "ex": f.Ex, + "is_pinned": f.IsPinned, + "version": f.Version, + "delete_time": f.DeleteTime, + } +} + +type Friend struct { + UserID string `bson:"user_id"` + Friends []*FriendElem `bson:"friends"` + Version uint `bson:"version"` + DeleteVersion uint `bson:"delete_version"` +} + +func (f *Friend) BuildDoc(lid any, e Elem) any { + return &Friend{ + UserID: lid.(string), + Friends: []*FriendElem{e.(*FriendElem)}, + } +} + +func (f *Friend) ElemsID() string { + return "user_id" +} + +func (f *Friend) IDName() string { + return "user_id" +} + +func (f *Friend) ElemsName() string { + return "friends" +} + +func (f *Friend) VersionName() string { + return "version" +} From c6942f07dae378fd3c367ef8fbcdbf9015bb6c03 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 May 2024 18:38:29 +0800 Subject: [PATCH 014/147] new mongo --- pkg/common/listdemo2/common.go | 163 ++++++++++++++++++++++++++++ pkg/common/listdemo2/common_test.go | 61 +++++++++++ pkg/common/listdemo2/demo.js | 86 +++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 pkg/common/listdemo2/common.go create mode 100644 pkg/common/listdemo2/common_test.go create mode 100644 pkg/common/listdemo2/demo.js diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go new file mode 100644 index 0000000000..081de8bbad --- /dev/null +++ b/pkg/common/listdemo2/common.go @@ -0,0 +1,163 @@ +package listdemo + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +var ( + ErrListNotFound = errors.New("list not found") + ErrElemExist = errors.New("elem exist") + ErrNeedFull = errors.New("need full") + ErrNotFound = mongo.ErrNoDocuments +) + +type Elem struct { + ID string + Version uint +} + +type ChangeLog struct { + ChangeIDs []Elem + DeleteIDs []Elem +} + +type WriteLog struct { + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` + DeleteVersion uint `bson:"delete_version"` +} + +type LogElem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + UpdateTime time.Time `bson:"update_time"` +} + +type LogModel struct { + coll *mongo.Collection +} + +func (l *LogModel) InitIndex(ctx context.Context) error { + return nil +} + +func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { + now := time.Now() + res, err := l.writeLog(ctx, dId, eId, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + wl := WriteLog{ + DID: dId, + Logs: []LogElem{ + { + EID: eId, + Deleted: deleted, + Version: 1, + UpdateTime: now, + }, + }, + Version: 1, + LastUpdate: now, + DeleteVersion: 0, + } + if _, err := l.coll.InsertOne(ctx, &wl); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLog(ctx, dId, eId, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eId", eId) + } + return nil +} + +func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + filter := bson.M{ + "d_id": dId, + } + elem := bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "update_time": now, + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "elem_index": bson.M{ + "$indexOfArray": []any{"$logs.e_id", eId}, + }, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "update_time": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$lt": []any{"$elem_index", 0}, + }, + "then": bson.M{ + "$concatArrays": []any{ + "$logs", + []bson.M{ + elem, + }, + }, + }, + "else": bson.M{ + "$map": bson.M{ + "input": bson.M{ + "$range": []any{0, bson.M{"$size": "$logs"}}, + }, + "as": "i", + "in": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$eq": []any{"$$i", "$elem_index"}, + }, + "then": elem, + "else": bson.M{ + "$arrayElemAt": []any{ + "$logs", + "$$i", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + "$unset": "elem_index", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint) (*ChangeLog, error) { + return nil, nil +} diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go new file mode 100644 index 0000000000..91f8f81436 --- /dev/null +++ b/pkg/common/listdemo2/common_test.go @@ -0,0 +1,61 @@ +package listdemo + +import ( + "context" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("demo") + _ = coll + //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + // { + // Keys: map[string]int{"user_id": 1}, + // }, + // { + // Keys: map[string]int{"friends.friend_user_id": 1}, + // }, + //})) + + wl := WriteLog{ + DID: "100", + Logs: []LogElem{ + { + EID: "1000", + Deleted: false, + Version: 1, + UpdateTime: time.Now(), + }, + { + EID: "2000", + Deleted: false, + Version: 1, + UpdateTime: time.Now(), + }, + }, + Version: 2, + DeleteVersion: 0, + LastUpdate: time.Now(), + } + + fmt.Println(Result(coll.InsertOne(context.Background(), wl))) + +} diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js new file mode 100644 index 0000000000..c0044b2387 --- /dev/null +++ b/pkg/common/listdemo2/demo.js @@ -0,0 +1,86 @@ +db.demo.updateMany( + { + "d_id": "100" + }, + [ + { + $addFields: { + elem_index: { + $indexOfArray: [ + "$logs.e_id", + "1000" + ] + } + } + }, + { + $set: { + version: { + $add: ["$version", 1] + }, + update_time: new Date(), + + } + }, + { + $set: { + logs: { + $cond: { + if: { + $lt: ["$elem_index", 0] + }, + then: { + $concatArrays: [ + "$logs", + [ + { + e_id: "1000", + update_time: new Date(), + version: "$version", + deleted: false + } + ] + ] + }, + else: { + $map: { + input: { + $range: [0, { + $size: "$logs" + }] + }, + as: "i", + in: { + $cond: { + if: { + $eq: ["$$i", "$elem_index"] + }, + then: { + e_id: "1000", + update_time: new Date(), + version: "$version", + deleted: false + }, + else: { + $arrayElemAt: ["$logs", "$$i"] + } + }, + + }, + + }, + + }, + + }, + + }, + + }, + + }, + { + $unset: ["elem_index"] + }, + ] +) From 82c6b005e93c88652d33123c24173b09ee86f949 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 23 May 2024 17:55:02 +0800 Subject: [PATCH 015/147] new mongo --- go.mod | 4 +- go.sum | 4 +- pkg/common/db/dataver/common.go | 250 ++++++++++++++++++++++++++++ pkg/common/db/mgo/friend.go | 62 +++++-- pkg/common/listdemo2/common.go | 239 ++++++++++++++++++++++++-- pkg/common/listdemo2/common_test.go | 42 +++-- pkg/common/listdemo2/demo.js | 8 +- pkg/common/listdemo2/demo2.js | 63 +++++++ pkg/common/listdemo2/demo3.js | 59 +++++++ pkg/common/listdemo2/demo5.js | 10 ++ 10 files changed, 677 insertions(+), 64 deletions(-) create mode 100644 pkg/common/db/dataver/common.go create mode 100644 pkg/common/listdemo2/demo2.js create mode 100644 pkg/common/listdemo2/demo3.js create mode 100644 pkg/common/listdemo2/demo5.js diff --git a/go.mod b/go.mod index 54e8a8e0e5..8e06e503fa 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.19 - github.com/pkg/errors v0.9.1 // indirect + github.com/openimsdk/tools v0.0.49-alpha.23 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 diff --git a/go.sum b/go.sum index 5611a6ca65..a80b2fb0a7 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= -github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= +github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go new file mode 100644 index 0000000000..c146600fb0 --- /dev/null +++ b/pkg/common/db/dataver/common.go @@ -0,0 +1,250 @@ +package dataver + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type WriteLog struct { + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` +} + +type Elem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type DataLog interface { + WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error +} + +func NewDataLog(coll *mongo.Collection) (DataLog, error) { + lm := &logModel{coll: coll} + if lm.initIndex(context.Background()) != nil { + return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) + } + return lm, nil +} + +type logModel struct { + coll *mongo.Collection +} + +func (l *logModel) initIndex(ctx context.Context) error { + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + }) + return err +} + +func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { + if len(eIds) == 0 { + return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + if datautil.Duplicate(eIds) { + return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + } + now := time.Now() + res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + if err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + } + return nil +} + +func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) error { + type tableWriteLog struct { + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + } + wl := tableWriteLog{ + DID: dId, + Logs: make([]Elem, 0, len(eIds)), + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, Elem{ + EID: eId, + Deleted: deleted, + Version: FirstVersion, + LastUpdate: now, + }) + } + _, err := l.coll.InsertOne(ctx, &wl) + return err +} + +func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *logModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": did, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + res, err := mongoutil.Aggregate[*WriteLog](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, errs.Wrap(mongo.ErrNoDocuments) + } + return res[0], nil +} + +func (l *logModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) +} diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 269bb594ab..6e7a6db980 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/tools/db/mongoutil" @@ -27,7 +28,9 @@ import ( // FriendMgo implements FriendModelInterface using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection + coll *mongo.Collection + owner dataver.DataLog + friend dataver.DataLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -43,12 +46,25 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { if err != nil { return nil, err } - return &FriendMgo{coll: coll}, nil + owner, err := dataver.NewDataLog(db.Collection("friend_owner_log")) + if err != nil { + return nil, err + } + friend, err := dataver.NewDataLog(db.Collection("friend_log")) + if err != nil { + return nil, err + } + return &FriendMgo{coll: coll, owner: owner, friend: friend}, nil } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) error { - return mongoutil.InsertMany(ctx, f.coll, friends) + return Success(func() error { + return mongoutil.InsertMany(ctx, f.coll, friends) + }, func() error { + + return nil + }) } // Delete removes specified friends of the owner user. @@ -57,7 +73,13 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.DeleteOne(ctx, f.coll, filter) + return Success(func() error { + return mongoutil.DeleteOne(ctx, f.coll, filter) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) + }, func() error { + + }) } // UpdateByMap updates specific fields of a friend document using a map. @@ -69,18 +91,13 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + return Success(func() error { + return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.FriendModel) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) @@ -157,7 +174,18 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - // Perform the update operation for all matching documents - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, update) - return err + return Success(func() error { + return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + }) +} + +func Success(fns ...func() error) error { + for _, fn := range fns { + if err := fn(); err != nil { + return err + } + } + return nil } diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go index 081de8bbad..2267d8630e 100644 --- a/pkg/common/listdemo2/common.go +++ b/pkg/common/listdemo2/common.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -17,6 +18,11 @@ var ( ErrNotFound = mongo.ErrNoDocuments ) +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + type Elem struct { ID string Version uint @@ -28,18 +34,27 @@ type ChangeLog struct { } type WriteLog struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` - DeleteVersion uint `bson:"delete_version"` + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +type WriteLogLen struct { + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` } type LogElem struct { EID string `bson:"e_id"` Deleted bool `bson:"deleted"` Version uint `bson:"version"` - UpdateTime time.Time `bson:"update_time"` + LastUpdate time.Time `bson:"last_update"` } type LogModel struct { @@ -47,7 +62,24 @@ type LogModel struct { } func (l *LogModel) InitIndex(ctx context.Context) error { - return nil + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + }) + return err +} + +func (l *LogModel) WriteLog1(ctx context.Context, dId string, eId string, deleted bool) { + if err := l.WriteLog(ctx, dId, eId, deleted); err != nil { + panic(err) + } +} + +func (l *LogModel) WriteLogBatch1(ctx context.Context, dId string, eIds []string, deleted bool) { + if err := l.WriteLogBatch(ctx, dId, eIds, deleted); err != nil { + panic(err) + } } func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { @@ -65,13 +97,13 @@ func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted { EID: eId, Deleted: deleted, - Version: 1, - UpdateTime: now, + Version: FirstVersion, + LastUpdate: now, }, }, - Version: 1, - LastUpdate: now, - DeleteVersion: 0, + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, } if _, err := l.coll.InsertOne(ctx, &wl); err == nil { return nil @@ -94,7 +126,7 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted "e_id": eId, "version": "$version", "deleted": deleted, - "update_time": now, + "last_update": now, } pipeline := []bson.M{ { @@ -107,7 +139,7 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted { "$set": bson.M{ "version": bson.M{"$add": []any{"$version", 1}}, - "update_time": now, + "last_update": now, }, }, { @@ -158,6 +190,181 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint) (*ChangeLog, error) { - return nil, nil +func (l *LogModel) WriteLogBatch(ctx context.Context, dId string, eIds []string, deleted bool) error { + if len(eIds) == 0 { + return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + if datautil.Duplicate(eIds) { + return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + } + now := time.Now() + res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + wl := WriteLog{ + DID: dId, + Logs: make([]LogElem, 0, len(eIds)), + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, LogElem{ + EID: eId, + Deleted: deleted, + Version: FirstVersion, + LastUpdate: now, + }) + } + if _, err := l.coll.InsertOne(ctx, &wl); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + } + return nil +} + +func (l *LogModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLogLen, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": did, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + res, err := mongoutil.Aggregate[*WriteLogLen](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, ErrNotFound + } + return res[0], nil +} + +func (l *LogModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) } diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go index 91f8f81436..bcaf4a19a6 100644 --- a/pkg/common/listdemo2/common_test.go +++ b/pkg/common/listdemo2/common_test.go @@ -2,7 +2,6 @@ package listdemo import ( "context" - "fmt" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "testing" @@ -24,7 +23,7 @@ func Check(err error) { func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("demo") + coll := cli.Database("openim_v3").Collection("friend_version") _ = coll //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ // { @@ -35,27 +34,24 @@ func TestName(t *testing.T) { // }, //})) - wl := WriteLog{ - DID: "100", - Logs: []LogElem{ - { - EID: "1000", - Deleted: false, - Version: 1, - UpdateTime: time.Now(), - }, - { - EID: "2000", - Deleted: false, - Version: 1, - UpdateTime: time.Now(), - }, - }, - Version: 2, - DeleteVersion: 0, - LastUpdate: time.Now(), - } + const num = 1 + lm := &LogModel{coll: coll} - fmt.Println(Result(coll.InsertOne(context.Background(), wl))) + //start := time.Now() + //eIds := make([]string, 0, num) + //for i := 0; i < num; i++ { + // eIds = append(eIds, strconv.Itoa(1000+(i))) + //} + //lm.WriteLogBatch1(context.Background(), "100", eIds, false) + //end := time.Now() + //t.Log(end.Sub(start)) // 509.962208ms + //t.Log(end.Sub(start) / num) // 511.496µs + start := time.Now() + wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) + if err != nil { + panic(err) + } + t.Log(time.Since(start)) + t.Log(wll) } diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js index c0044b2387..be9c74cb98 100644 --- a/pkg/common/listdemo2/demo.js +++ b/pkg/common/listdemo2/demo.js @@ -1,4 +1,4 @@ -db.demo.updateMany( +db.friend_version.updateMany( { "d_id": "100" }, @@ -18,7 +18,7 @@ db.demo.updateMany( version: { $add: ["$version", 1] }, - update_time: new Date(), + last_update: new Date(), } }, @@ -35,7 +35,7 @@ db.demo.updateMany( [ { e_id: "1000", - update_time: new Date(), + last_update: new Date(), version: "$version", deleted: false } @@ -57,7 +57,7 @@ db.demo.updateMany( }, then: { e_id: "1000", - update_time: new Date(), + last_update: new Date(), version: "$version", deleted: false }, diff --git a/pkg/common/listdemo2/demo2.js b/pkg/common/listdemo2/demo2.js new file mode 100644 index 0000000000..15e7abcfe0 --- /dev/null +++ b/pkg/common/listdemo2/demo2.js @@ -0,0 +1,63 @@ + +db.friend_version.updateMany( + { + "d_id": "100" + }, + [ + { + $addFields: { + update_elem_ids: ["1000", "1001","1003", "2000"] + } + }, + { + $set: { + version: { + $add: ["$version", 1] + }, + last_update: new Date(), + } + }, + { + $set: { + logs: { + $filter: { + input: "$logs", + as: "log", + cond: { + "$not": { + $in: ["$$log.e_id", "$update_elem_ids"] + } + } + } + }, + + }, + + }, + { + $set: { + logs: { + $concatArrays: [ + "$logs", + [ + { + e_id: "1003", + last_update: ISODate("2024-05-25T06:32:10.238Z"), + version: "$version", + deleted: false + }, + + ] + ] + } + } + }, + { + $unset: ["update_elem_ids"] + }, + + ] +) + + + diff --git a/pkg/common/listdemo2/demo3.js b/pkg/common/listdemo2/demo3.js new file mode 100644 index 0000000000..5367971cf9 --- /dev/null +++ b/pkg/common/listdemo2/demo3.js @@ -0,0 +1,59 @@ + +db.friend_version.aggregate([ + { + "$match": { + "d_id": "100", + } + }, + { + "$project": { + "_id": 0, + "d_id": 0, + } + }, + { + "$addFields": { + "logs": { + $cond: { + if: { + $or: [ + {$lt: ["$version", 3]}, + {$gte: ["$deleted", 3]}, + ], + }, + then: [], + else: "$logs", + } + } + }, + }, + { + "$addFields": { + "logs": { + "$filter": { + input: "$logs", + as: "l", + cond: { $gt: ["$$l.version", 3] } + } + } + } + }, + { + "$addFields": { + "log_len": { + $size: "$logs" + } + } + }, + { + "$addFields": { + "logs": { + $cond: { + if: {$gt: ["$log_len", 1]}, + then: [], + else: "$logs", + } + } + } + } +]) diff --git a/pkg/common/listdemo2/demo5.js b/pkg/common/listdemo2/demo5.js new file mode 100644 index 0000000000..71a709a811 --- /dev/null +++ b/pkg/common/listdemo2/demo5.js @@ -0,0 +1,10 @@ +db.friend_version.updateMany( + { + "d_id": "100" + }, + [ + + + + ], +) \ No newline at end of file From a2bfc907bb9e68c53b63c9183ff0887a635b0fd7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 23 May 2024 18:39:45 +0800 Subject: [PATCH 016/147] new mongo --- pkg/common/db/mgo/friend.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 6e7a6db980..b82d1c3aba 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -62,7 +62,26 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) return Success(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { - + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) + } + for ownerUserID, friendUserIDs := range mp { + if err := f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false); err != nil { + return err + } + } + return nil + }, func() error { + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.FriendUserID] = append(mp[friend.FriendUserID], friend.OwnerUserID) + } + for friendUserID, ownerUserIDs := range mp { + if err := f.friend.WriteLog(ctx, friendUserID, ownerUserIDs, false); err != nil { + return err + } + } return nil }) } @@ -78,7 +97,12 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) }, func() error { - + for _, userID := range friendUserIDs { + if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { + return err + } + } + return nil }) } @@ -95,6 +119,8 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + }, func() error { + return f.friend.WriteLog(ctx, friendUserID, []string{ownerUserID}, false) }) } @@ -178,6 +204,13 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + }, func() error { + for _, userID := range friendUserIDs { + if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { + return err + } + } + return nil }) } From 05cec1b10c8e88dccd24be65839ac4c29bc365d9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 09:54:17 +0800 Subject: [PATCH 017/147] new mongo --- pkg/common/db/mgo/friend.go | 38 +++---------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index b82d1c3aba..a44244fcad 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -28,9 +28,8 @@ import ( // FriendMgo implements FriendModelInterface using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection - owner dataver.DataLog - friend dataver.DataLog + coll *mongo.Collection + owner dataver.DataLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -50,11 +49,7 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { if err != nil { return nil, err } - friend, err := dataver.NewDataLog(db.Collection("friend_log")) - if err != nil { - return nil, err - } - return &FriendMgo{coll: coll, owner: owner, friend: friend}, nil + return &FriendMgo{coll: coll, owner: owner}, nil } // Create inserts multiple friend records. @@ -72,17 +67,6 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) } } return nil - }, func() error { - mp := make(map[string][]string) - for _, friend := range friends { - mp[friend.FriendUserID] = append(mp[friend.FriendUserID], friend.OwnerUserID) - } - for friendUserID, ownerUserIDs := range mp { - if err := f.friend.WriteLog(ctx, friendUserID, ownerUserIDs, false); err != nil { - return err - } - } - return nil }) } @@ -96,13 +80,6 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) - }, func() error { - for _, userID := range friendUserIDs { - if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { - return err - } - } - return nil }) } @@ -119,8 +96,6 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) - }, func() error { - return f.friend.WriteLog(ctx, friendUserID, []string{ownerUserID}, false) }) } @@ -204,13 +179,6 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) - }, func() error { - for _, userID := range friendUserIDs { - if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { - return err - } - } - return nil }) } From 11840025fc1f44808df4fbf669860c3f81e6c0af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 15:05:12 +0800 Subject: [PATCH 018/147] new mongo --- pkg/common/db/dataver/common.go | 23 +++++++++++++-- pkg/common/db/dataver/result.go | 32 +++++++++++++++++++++ pkg/common/db/mgo/friend.go | 15 ++++++++++ pkg/common/listdemo2/common.go | 50 +-------------------------------- 4 files changed, 68 insertions(+), 52 deletions(-) create mode 100644 pkg/common/db/dataver/result.go diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c146600fb0..c7e4658ab4 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -24,6 +24,23 @@ type WriteLog struct { LogLen int `bson:"log_len"` } +func (w *WriteLog) Full() bool { + if w.Version == 0 { + return true + } + return len(w.Logs) != w.LogLen +} + +func (w *WriteLog) DeleteEId() []string { + var eIds []string + for _, l := range w.Logs { + if l.Deleted { + eIds = append(eIds, l.EID) + } + } + return eIds +} + type Elem struct { EID string `bson:"e_id"` Deleted bool `bson:"deleted"` @@ -33,7 +50,7 @@ type Elem struct { type DataLog interface { WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error - FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } @@ -173,11 +190,11 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *logModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) { +func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { pipeline := []bson.M{ { "$match": bson.M{ - "d_id": did, + "d_id": dId, }, }, { diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go new file mode 100644 index 0000000000..b6e6d22b46 --- /dev/null +++ b/pkg/common/db/dataver/result.go @@ -0,0 +1,32 @@ +package dataver + +type SyncResult[T any] struct { + Version uint + DeleteEID []string + Changes []T + Full bool +} + +func NewSyncResult[T any](wl *WriteLog, find func(eIds []string) ([]T, error)) (*SyncResult[T], error) { + var findEIDs []string + var res SyncResult[T] + if wl.Full() { + res.Full = true + } else { + for _, l := range wl.Logs { + if l.Deleted { + res.DeleteEID = append(res.DeleteEID, l.EID) + } else { + findEIDs = append(findEIDs, l.EID) + } + } + } + if res.Full || len(findEIDs) > 0 { + var err error + res.Changes, err = find(findEIDs) + if err != nil { + return nil, err + } + } + return &res, nil +} diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index a44244fcad..78e74edd33 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" @@ -182,6 +183,20 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien }) } +func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { + res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) + if err != nil { + return nil, err + } + return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { + if len(eIds) == 0 { + return nil, errors.New("todo") + } else { + return f.FindFriends(ctx, ownerUserID, eIds) + } + }) +} + func Success(fns ...func() error) error { for _, fn := range fns { if err := fn(); err != nil { diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go index 2267d8630e..40419d533c 100644 --- a/pkg/common/listdemo2/common.go +++ b/pkg/common/listdemo2/common.go @@ -70,54 +70,6 @@ func (l *LogModel) InitIndex(ctx context.Context) error { return err } -func (l *LogModel) WriteLog1(ctx context.Context, dId string, eId string, deleted bool) { - if err := l.WriteLog(ctx, dId, eId, deleted); err != nil { - panic(err) - } -} - -func (l *LogModel) WriteLogBatch1(ctx context.Context, dId string, eIds []string, deleted bool) { - if err := l.WriteLogBatch(ctx, dId, eIds, deleted); err != nil { - panic(err) - } -} - -func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { - now := time.Now() - res, err := l.writeLog(ctx, dId, eId, deleted, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil - } - wl := WriteLog{ - DID: dId, - Logs: []LogElem{ - { - EID: eId, - Deleted: deleted, - Version: FirstVersion, - LastUpdate: now, - }, - }, - Version: FirstVersion, - Deleted: DefaultDeleteVersion, - LastUpdate: now, - } - if _, err := l.coll.InsertOne(ctx, &wl); err == nil { - return nil - } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLog(ctx, dId, eId, deleted, now); err != nil { - return err - } else if res.ModifiedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eId", eId) - } - return nil -} - func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { filter := bson.M{ "d_id": dId, @@ -356,7 +308,7 @@ func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, return nil, err } if len(res) == 0 { - return nil, ErrNotFound + return &WriteLogLen{}, nil } return res[0], nil } From e99eaf91464d568e806ae4fcecba4df75a2ee633 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 15:44:08 +0800 Subject: [PATCH 019/147] new mongo --- pkg/common/db/dataver/common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c7e4658ab4..d5a0c16949 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -97,7 +97,7 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele } if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { return err - } else if res.ModifiedCount == 0 { + } else if res.MatchedCount == 0 { return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) } return nil @@ -253,7 +253,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, return nil, err } if len(res) == 0 { - return nil, errs.Wrap(mongo.ErrNoDocuments) + return &WriteLog{}, nil } return res[0], nil } From 2f97933f6ca841f416aeceff188f6116e8e94fb1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 18:11:48 +0800 Subject: [PATCH 020/147] new mongo --- pkg/common/db/dataver/common.go | 14 ++++++------ pkg/common/db/dataver/todo.go | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 pkg/common/db/dataver/todo.go diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index d5a0c16949..bc367d795b 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -6,6 +6,7 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "time" ) @@ -16,12 +17,13 @@ const ( ) type WriteLog struct { - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` } func (w *WriteLog) Full() bool { diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go new file mode 100644 index 0000000000..69b979fe5a --- /dev/null +++ b/pkg/common/db/dataver/todo.go @@ -0,0 +1,38 @@ +package dataver + +/* + +UserIDs 顺序 +前500顺序 + + +1,2,3,4,5,6,7,8,9 + +1,3,5,7,8,9 + + +1.sdk添加一个表记录 docID(后续换名字), version +2.sdk同步，先计算idHash，api调用参数idHash, docID, version +3.服务器先判断version变更记录，没有直接返回同步成功。 + 有变更，先查版本变更记录，在查前500id，变更记录只保留前500id中的 + 根据前500id计算idHash，不一致返回会全量id，不反悔删除id + 全量同步有标识，只返回全量id + 变更记录只包含id，不包括详细信息。 +4.sdk通过变更记录，同步数据不一致进行重试。 + + + + + + + + + + + + + + + + +*/ From 1f02bdc26779b0c56a57d3dd0e00f534538834ec Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 27 May 2024 15:15:23 +0800 Subject: [PATCH 021/147] friend incr sync --- go.mod | 6 ++- go.sum | 4 +- internal/rpc/friend/convert.go | 26 +++++++++++ internal/rpc/friend/sync.go | 65 ++++++++++++++++++++++++++ pkg/common/cachekey/friend.go | 15 ++++-- pkg/common/config/config.go | 3 +- pkg/common/db/cache/friend.go | 33 +++++++++++++ pkg/common/db/controller/friend.go | 21 +++++++-- pkg/common/db/dataver/common.go | 16 +++++++ pkg/common/db/dataver/result.go | 25 +++++----- pkg/common/db/dataver/todo.go | 2 +- pkg/common/db/mgo/friend.go | 50 ++++++++++++-------- pkg/common/db/table/relation/friend.go | 7 +++ pkg/common/listdemo2/common_test.go | 48 ++++++++++--------- 14 files changed, 253 insertions(+), 68 deletions(-) create mode 100644 internal/rpc/friend/convert.go create mode 100644 internal/rpc/friend/sync.go diff --git a/go.mod b/go.mod index 8e06e503fa..c3d69b0fbf 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.65 + github.com/openimsdk/protocol v0.0.69-alpha.1 github.com/openimsdk/tools v0.0.49-alpha.23 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 @@ -177,3 +177,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index a80b2fb0a7..908c15c871 100644 --- a/go.sum +++ b/go.sum @@ -286,8 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= -github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.1 h1:l/PN8mwmh5O7PRoaiMZvey+hUxyuNNnMgPGKOfEMOKs= +github.com/openimsdk/protocol v0.0.69-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go new file mode 100644 index 0000000000..aa68952843 --- /dev/null +++ b/internal/rpc/friend/convert.go @@ -0,0 +1,26 @@ +package friend + +import ( + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/tools/utils/datautil" +) + +func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { + return &friend.FriendInfo{ + OwnerUserID: db.OwnerUserID, + FriendUserID: db.FriendUserID, + FriendNickname: db.FriendNickname, + FriendFaceURL: db.FriendFaceURL, + Remark: db.Remark, + CreateTime: db.CreateTime.UnixMilli(), + AddSource: db.AddSource, + OperatorUserID: db.OperatorUserID, + Ex: db.Ex, + IsPinned: db.IsPinned, + } +} + +func friendsDB2PB(db []*relationtb.FriendModel) []*friend.FriendInfo { + return datautil.Slice(db, friendDB2PB) +} diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go new file mode 100644 index 0000000000..1e74cbc0db --- /dev/null +++ b/internal/rpc/friend/sync.go @@ -0,0 +1,65 @@ +package friend + +import ( + "context" + "crypto/md5" + "encoding/binary" + "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + pbfriend "github.com/openimsdk/protocol/friend" +) + +func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { + data, _ := json.Marshal(userIDs) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} + +func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSyncFriendsReq) (*pbfriend.IncrSyncFriendsResp, error) { + var limit int + if req.Version > 0 { + limit = s.config.RpcConfig.FriendSyncCount + } + incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) + if err != nil { + return nil, err + } + sortUserIDs, err := s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + if len(sortUserIDs) == 0 { + return &pbfriend.IncrSyncFriendsResp{ + Version: uint64(incrVer.Version), + Full: true, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + }, nil + } + var changes []*relation.FriendModel + res := dataver.NewSyncResult(incrVer, sortUserIDs) + if len(res.Changes) > 0 { + changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) + if err != nil { + return nil, err + } + } + calcHash := s.sortFriendUserIDsHash(sortUserIDs) + if calcHash == req.IdHash { + sortUserIDs = nil + } + return &pbfriend.IncrSyncFriendsResp{ + Version: uint64(res.Version), + Full: res.Full, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + SortUserIdHash: calcHash, + SortUserIds: sortUserIDs, + DeleteUserIds: res.DeleteEID, + Changes: friendsDB2PB(changes), + }, nil +} diff --git a/pkg/common/cachekey/friend.go b/pkg/common/cachekey/friend.go index 9691b1f5c9..6a217bdef9 100644 --- a/pkg/common/cachekey/friend.go +++ b/pkg/common/cachekey/friend.go @@ -14,11 +14,14 @@ package cachekey +import "strconv" + const ( - FriendIDsKey = "FRIEND_IDS:" - TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" - FriendKey = "FRIEND_INFO:" - IsFriendKey = "IS_FRIEND:" // local cache key + FriendIDsKey = "FRIEND_IDS:" + TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" + FriendKey = "FRIEND_INFO:" + IsFriendKey = "IS_FRIEND:" // local cache key + FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -36,3 +39,7 @@ func GetFriendKey(ownerUserID, friendUserID string) string { func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } + +func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { + return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 12c4f7f789..881e055987 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -244,7 +244,8 @@ type Friend struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + FriendSyncCount int `mapstructure:"friendSyncCount"` } type Group struct { diff --git a/pkg/common/db/cache/friend.go b/pkg/common/db/cache/friend.go index 73fe5ea694..4a51a8c178 100644 --- a/pkg/common/db/cache/friend.go +++ b/pkg/common/db/cache/friend.go @@ -16,6 +16,7 @@ package cache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/dtm-labs/rockscache" @@ -41,12 +42,19 @@ type FriendCache interface { GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) // Called when friendID list changed DelFriendIDs(ownerUserID ...string) FriendCache + + DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + // Get single friendInfo from the cache GetFriend(ctx context.Context, ownerUserID, friendUserID string) (friend *relationtb.FriendModel, err error) // Delete friend when friend info changed DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } // FriendCacheRedis is an implementation of the FriendCache interface using Redis. @@ -55,6 +63,7 @@ type FriendCacheRedis struct { friendDB relationtb.FriendModelInterface expireTime time.Duration rcClient *rockscache.Client + syncCount int } // NewFriendCacheRedis creates a new instance of FriendCacheRedis. @@ -89,6 +98,10 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } +func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { + return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +} + // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string { return cachekey.GetTwoWayFriendsIDsKey(ownerUserID) @@ -118,6 +131,16 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) FriendCache { return newGroupCache } +func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache { + newGroupCache := f.NewCache() + keys := make([]string, 0, len(ownerUserIDs)) + for _, userID := range ownerUserIDs { + keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) + } + newGroupCache.AddKeys(keys...) + return newGroupCache +} + // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { friendIDs, err := f.GetFriendIDs(ctx, ownerUserID) @@ -172,3 +195,13 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } + +func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { + return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { + return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) + }) +} + +func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) +} diff --git a/pkg/common/db/controller/friend.go b/pkg/common/db/controller/friend.go index 49136f2288..49c0cb990e 100644 --- a/pkg/common/db/controller/friend.go +++ b/pkg/common/db/controller/friend.go @@ -17,6 +17,7 @@ package controller import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" @@ -75,6 +76,10 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } type friendDatabase struct { @@ -173,7 +178,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return err } newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...) + cache = cache.DelFriendIDs(newFriendIDs...).DelSortFriendUserIDs(ownerUserID) return cache.ExecDel(ctx) }) @@ -276,7 +281,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) }) } @@ -285,7 +290,7 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).ExecDel(ctx) + return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).DelSortFriendUserIDs(ownerUserID).ExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -342,5 +347,13 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { return err } - return f.cache.DelFriends(ownerUserID, friendUserIDs).ExecDel(ctx) + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelSortFriendUserIDs(ownerUserID).ExecDel(ctx) +} + +func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { + return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +} + +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index bc367d795b..c9a4e0419f 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -2,12 +2,14 @@ package dataver import ( "context" + "errors" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "time" ) @@ -192,7 +194,21 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } +func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { + res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err == nil { + return res, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return &WriteLog{}, nil + } else { + return nil, err + } +} + func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { + if version == 0 && limit == 0 { + return l.findDoc(ctx, dId) + } pipeline := []bson.M{ { "$match": bson.M{ diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go index b6e6d22b46..915a618d0e 100644 --- a/pkg/common/db/dataver/result.go +++ b/pkg/common/db/dataver/result.go @@ -1,32 +1,31 @@ package dataver -type SyncResult[T any] struct { +import "github.com/openimsdk/tools/utils/datautil" + +type SyncResult struct { Version uint DeleteEID []string - Changes []T + Changes []string Full bool } -func NewSyncResult[T any](wl *WriteLog, find func(eIds []string) ([]T, error)) (*SyncResult[T], error) { +func NewSyncResult(wl *WriteLog, fullIDs []string) *SyncResult { var findEIDs []string - var res SyncResult[T] + var res SyncResult if wl.Full() { + res.Changes = fullIDs res.Full = true } else { + idSet := datautil.SliceSet(fullIDs) for _, l := range wl.Logs { if l.Deleted { res.DeleteEID = append(res.DeleteEID, l.EID) } else { - findEIDs = append(findEIDs, l.EID) + if _, ok := idSet[l.EID]; ok { + findEIDs = append(findEIDs, l.EID) + } } } } - if res.Full || len(findEIDs) > 0 { - var err error - res.Changes, err = find(findEIDs) - if err != nil { - return nil, err - } - } - return &res, nil + return &res } diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go index 69b979fe5a..cd2be14f01 100644 --- a/pkg/common/db/dataver/todo.go +++ b/pkg/common/db/dataver/todo.go @@ -19,7 +19,7 @@ UserIDs 顺序 全量同步有标识，只返回全量id 变更记录只包含id，不包括详细信息。 4.sdk通过变更记录，同步数据不一致进行重试。 - +5.先修改db，在自增版本号，外层加事务 diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 78e74edd33..bca6873c28 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,7 +16,6 @@ package mgo import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" @@ -55,7 +54,7 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) error { - return Success(func() error { + return IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { mp := make(map[string][]string) @@ -77,7 +76,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return Success(func() error { + return IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) @@ -93,7 +92,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return Success(func() error { + return IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) @@ -146,7 +145,14 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}) + return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination, opt) +} + +func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { + filter := bson.M{"owner_user_id": ownerUserID} + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}).SetLimit(int64(limit)) + return mongoutil.Find[string](ctx, f.coll, filter, opt) } // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. @@ -176,29 +182,33 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - return Success(func() error { + return IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) }) } -func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { - res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) - if err != nil { - return nil, err - } - return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { - if len(eIds) == 0 { - return nil, errors.New("todo") - } else { - return f.FindFriends(ctx, ownerUserID, eIds) - } - }) +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } -func Success(fns ...func() error) error { - for _, fn := range fns { +//func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { +// res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) +// if err != nil { +// return nil, err +// } +// return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { +// if len(eIds) == 0 { +// return nil, errors.New("todo") +// } else { +// return f.FindFriends(ctx, ownerUserID, eIds) +// } +// }) +//} + +func IncrVersion(dbs ...func() error) error { + for _, fn := range dbs { if err := fn(); err != nil { return err } diff --git a/pkg/common/db/table/relation/friend.go b/pkg/common/db/table/relation/friend.go index 4c84e773dd..a28b2278d3 100644 --- a/pkg/common/db/table/relation/friend.go +++ b/pkg/common/db/table/relation/friend.go @@ -16,6 +16,7 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/openimsdk/tools/db/pagination" @@ -25,6 +26,8 @@ import ( type FriendModel struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` + FriendNickname string `bson:"friend_nickname"` + FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` @@ -53,10 +56,14 @@ type FriendModelInterface interface { FindReversalFriends(ctx context.Context, friendUserID string, ownerUserIDs []string) (friends []*FriendModel, err error) // FindOwnerFriends retrieves a paginated list of friends for a given owner. FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, friends []*FriendModel, err error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (total int64, friends []*FriendModel, err error) // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. FindFriendUserIDs(ctx context.Context, ownerUserID string) (friendUserIDs []string, err error) // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go index bcaf4a19a6..c03cc31bd2 100644 --- a/pkg/common/listdemo2/common_test.go +++ b/pkg/common/listdemo2/common_test.go @@ -2,10 +2,9 @@ package listdemo import ( "context" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" + "errors" + "github.com/openimsdk/tools/db/mongoutil" "testing" - "time" ) func Result[V any](val V, err error) V { @@ -22,20 +21,25 @@ func Check(err error) { } func TestName(t *testing.T) { - cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("friend_version") - _ = coll - //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ - // { - // Keys: map[string]int{"user_id": 1}, - // }, - // { - // Keys: map[string]int{"friends.friend_user_id": 1}, - // }, - //})) + cli := Result(mongoutil.NewMongoDB(context.Background(), &mongoutil.Config{Uri: "mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100", Database: "openim_v3"})) + + db := cli.GetDB() + tx := cli.GetTx() const num = 1 - lm := &LogModel{coll: coll} + lm := &LogModel{coll: db.Collection("friend_version")} + + err := tx.Transaction(context.Background(), func(ctx context.Context) error { + err := tx.Transaction(ctx, func(ctx context.Context) error { + return lm.WriteLogBatch(ctx, "100", []string{"1000", "2000"}, true) + }) + if err != nil { + t.Log("--------->") + return err + } + return errors.New("1234") + }) + t.Log(err) //start := time.Now() //eIds := make([]string, 0, num) @@ -47,11 +51,11 @@ func TestName(t *testing.T) { //t.Log(end.Sub(start)) // 509.962208ms //t.Log(end.Sub(start) / num) // 511.496µs - start := time.Now() - wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) - if err != nil { - panic(err) - } - t.Log(time.Since(start)) - t.Log(wll) + //start := time.Now() + //wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) + //if err != nil { + // panic(err) + //} + //t.Log(time.Since(start)) + //t.Log(wll) } From 8e37a417db65c8bb666caab6dda714340356a396 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 27 May 2024 18:24:33 +0800 Subject: [PATCH 022/147] friend incr sync --- go.mod | 6 ++-- internal/api/friend.go | 5 +++ internal/api/router.go | 1 + internal/rpc/friend/sync.go | 14 +++++--- internal/rpc/group/sync.go | 16 +++++++++ internal/rpc/user/user.go | 5 +++ pkg/common/db/dataver/common.go | 61 +++++++++++++++++++++++---------- pkg/common/db/dataver/result.go | 17 +++++++-- pkg/common/db/mgo/friend.go | 14 -------- 9 files changed, 96 insertions(+), 43 deletions(-) create mode 100644 internal/rpc/group/sync.go diff --git a/go.mod b/go.mod index c3d69b0fbf..daf6cc0d24 100644 --- a/go.mod +++ b/go.mod @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +replace ( + github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +) diff --git a/internal/api/friend.go b/internal/api/friend.go index 1fea38b313..3af162a536 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -90,6 +90,11 @@ func (o *FriendApi) GetFriendIDs(c *gin.Context) { func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } + func (o *FriendApi) UpdateFriends(c *gin.Context) { a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) } + +func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { + a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 6005671785..78e049e0f3 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -86,6 +86,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) + friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 1e74cbc0db..9d2a53b586 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "encoding/binary" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" @@ -21,7 +22,10 @@ func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { return binary.BigEndian.Uint64(sum[:]) } -func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSyncFriendsReq) (*pbfriend.IncrSyncFriendsResp, error) { +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } var limit int if req.Version > 0 { limit = s.config.RpcConfig.FriendSyncCount @@ -35,14 +39,15 @@ func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSy return nil, err } if len(sortUserIDs) == 0 { - return &pbfriend.IncrSyncFriendsResp{ + return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(incrVer.Version), + VersionID: dataver.VersionIDStr(incrVer.ID), Full: true, SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), }, nil } var changes []*relation.FriendModel - res := dataver.NewSyncResult(incrVer, sortUserIDs) + res := dataver.NewSyncResult(incrVer, sortUserIDs, req.VersionID) if len(res.Changes) > 0 { changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) if err != nil { @@ -53,8 +58,9 @@ func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSy if calcHash == req.IdHash { sortUserIDs = nil } - return &pbfriend.IncrSyncFriendsResp{ + return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(res.Version), + VersionID: res.VersionID, Full: res.Full, SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), SortUserIdHash: calcHash, diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go new file mode 100644 index 0000000000..765bceb632 --- /dev/null +++ b/internal/rpc/group/sync.go @@ -0,0 +1,16 @@ +package group + +import ( + "context" + pbgroup "github.com/openimsdk/protocol/group" +) + +func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *groupServer) GetGroupMemberHash(ctx context.Context, req *pbgroup.GetGroupMemberHashReq) (*pbgroup.GetGroupMemberHashResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index a28fa24e28..2296ee62fa 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -56,6 +56,11 @@ type userServer struct { webhookClient *webhook.Client } +func (s *userServer) SearchUser(ctx context.Context, req *pbuser.SearchUserReq) (*pbuser.SearchUserResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.User RedisConfig config.Redis diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c9a4e0419f..3ff9906e54 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -52,6 +52,27 @@ type Elem struct { LastUpdate time.Time `bson:"last_update"` } +type tableWriteLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (t *tableWriteLog) WriteLog() *WriteLog { + return &WriteLog{ + ID: t.ID, + DID: t.DID, + Logs: t.Logs, + Version: t.Version, + Deleted: t.Deleted, + LastUpdate: t.LastUpdate, + LogLen: 0, + } +} + type DataLog interface { WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) @@ -94,7 +115,7 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele if res.MatchedCount > 0 { return nil } - if err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + if _, err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { return nil } else if !mongo.IsDuplicateKeyError(err) { return err @@ -107,15 +128,9 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele return nil } -func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) error { - type tableWriteLog struct { - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - } +func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*tableWriteLog, error) { wl := tableWriteLog{ + ID: primitive.NewObjectID(), DID: dId, Logs: make([]Elem, 0, len(eIds)), Version: FirstVersion, @@ -131,12 +146,12 @@ func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, delet }) } _, err := l.coll.InsertOne(ctx, &wl) - return err + return &wl, err } func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - if len(eIds) == 0 { - return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + if eIds == nil { + eIds = []string{} } filter := bson.M{ "d_id": dId, @@ -195,17 +210,25 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, } func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) - if err == nil { - return res, nil - } else if errors.Is(err, mongo.ErrNoDocuments) { - return &WriteLog{}, nil + return mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) +} + +func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { + if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { + return wl, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { + return res.WriteLog(), nil + } else if mongo.IsDuplicateKeyError(err) { + return l.findChangeLog(ctx, dId, version, limit) } else { return nil, err } } -func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { if version == 0 && limit == 0 { return l.findDoc(ctx, dId) } @@ -271,7 +294,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, return nil, err } if len(res) == 0 { - return &WriteLog{}, nil + return nil, mongo.ErrNoDocuments } return res[0], nil } diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go index 915a618d0e..ec09ffa314 100644 --- a/pkg/common/db/dataver/result.go +++ b/pkg/common/db/dataver/result.go @@ -1,18 +1,29 @@ package dataver -import "github.com/openimsdk/tools/utils/datautil" +import ( + "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson/primitive" +) type SyncResult struct { Version uint + VersionID string DeleteEID []string Changes []string Full bool } -func NewSyncResult(wl *WriteLog, fullIDs []string) *SyncResult { +func VersionIDStr(id primitive.ObjectID) string { + if id.IsZero() { + return "" + } + return id.String() +} + +func NewSyncResult(wl *WriteLog, fullIDs []string, versionID string) *SyncResult { var findEIDs []string var res SyncResult - if wl.Full() { + if wl.Full() || VersionIDStr(wl.ID) != versionID { res.Changes = fullIDs res.Full = true } else { diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index bca6873c28..37ba2d7ac2 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -193,20 +193,6 @@ func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, ver return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } -//func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { -// res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) -// if err != nil { -// return nil, err -// } -// return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { -// if len(eIds) == 0 { -// return nil, errors.New("todo") -// } else { -// return f.FindFriends(ctx, ownerUserID, eIds) -// } -// }) -//} - func IncrVersion(dbs ...func() error) error { for _, fn := range dbs { if err := fn(); err != nil { From f6131c4ce578674777baab10f6730bf4a5194ec4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 29 May 2024 15:13:50 +0800 Subject: [PATCH 023/147] friend incr sync --- internal/api/friend.go | 17 ++++++++++++ internal/rpc/friend/sync.go | 48 ++++++++++++++------------------- pkg/common/db/dataver/common.go | 9 ++++--- pkg/common/db/mgo/mongo_test.go | 34 +++++++++++++++++++++++ 4 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 pkg/common/db/mgo/mongo_test.go diff --git a/internal/api/friend.go b/internal/api/friend.go index 3af162a536..5b2dfa0ca7 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -98,3 +98,20 @@ func (o *FriendApi) UpdateFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) } + +//func BatchIncremental[A, B, C any,D comparable](c *gin.Context, rpc func(client C, ctx context.Context, req *A, options ...grpc.CallOption) (*B, error), getID func(req *A)D, setID func(req *A, id D)) { +// req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq[A]](c) +// if err != nil { +// apiresp.GinError(c, err) +// return +// } +// if len(req.List) == 0 { +// apiresp.GinError(c, errs.ErrArgs.WrapMsg("empty versions list")) +// return +// } +//} +// +//type BatchIncrementalReq[A any] struct { +// UserID string `json:"user_id"` +// List []*A `json:"list"` +//} diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 9d2a53b586..a2f49b283f 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "encoding/json" "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" ) @@ -34,38 +33,31 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. if err != nil { return nil, err } - sortUserIDs, err := s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - if err != nil { - return nil, err - } - if len(sortUserIDs) == 0 { - return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(incrVer.Version), - VersionID: dataver.VersionIDStr(incrVer.ID), - Full: true, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - }, nil - } - var changes []*relation.FriendModel - res := dataver.NewSyncResult(incrVer, sortUserIDs, req.VersionID) - if len(res.Changes) > 0 { - changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) + var ( + deleteUserIDs []string + changeUserIDs []string + ) + if incrVer.Full() { + changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } + } else { + deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() } - calcHash := s.sortFriendUserIDsHash(sortUserIDs) - if calcHash == req.IdHash { - sortUserIDs = nil + var friends []*relation.FriendModel + if len(changeUserIDs) > 0 { + friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) + if err != nil { + return nil, err + } } return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(res.Version), - VersionID: res.VersionID, - Full: res.Full, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - SortUserIdHash: calcHash, - SortUserIds: sortUserIDs, - DeleteUserIds: res.DeleteEID, - Changes: friendsDB2PB(changes), + Version: uint64(incrVer.Version), + VersionID: incrVer.ID.String(), + Full: incrVer.Full(), + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + DeleteUserIds: deleteUserIDs, + Changes: friendsDB2PB(friends), }, nil } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index 3ff9906e54..e29ab2b967 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -35,14 +35,15 @@ func (w *WriteLog) Full() bool { return len(w.Logs) != w.LogLen } -func (w *WriteLog) DeleteEId() []string { - var eIds []string +func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { for _, l := range w.Logs { if l.Deleted { - eIds = append(eIds, l.EID) + delIds = append(delIds, l.EID) + } else { + changeIds = append(changeIds, l.EID) } } - return eIds + return } type Elem struct { diff --git a/pkg/common/db/mgo/mongo_test.go b/pkg/common/db/mgo/mongo_test.go new file mode 100644 index 0000000000..294dd13173 --- /dev/null +++ b/pkg/common/db/mgo/mongo_test.go @@ -0,0 +1,34 @@ +package mgo + +import ( + "context" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + conv := Result(NewConversationMongo(cli.Database("openim_v3"))) + num, err := conv.GetAllConversationIDsNumber(context.Background()) + if err != nil { + panic(err) + } + t.Log(num) + ids := Result(conv.GetAllConversationIDs(context.Background())) + t.Log(ids) +} From 0d57f28a3a523e9d819a661fbbc432b2921c9adc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 29 May 2024 15:20:17 +0800 Subject: [PATCH 024/147] friend incr sync --- go.mod | 8 ++++---- go.sum | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index daf6cc0d24..0b2f3cb3b1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.1 + github.com/openimsdk/protocol v0.0.69-alpha.2 github.com/openimsdk/tools v0.0.49-alpha.23 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace ( - github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -) +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index 908c15c871..c0bd78645e 100644 --- a/go.sum +++ b/go.sum @@ -286,8 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.1 h1:l/PN8mwmh5O7PRoaiMZvey+hUxyuNNnMgPGKOfEMOKs= -github.com/openimsdk/protocol v0.0.69-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.2 h1:XNc3pmAXyW+PMo7tghr2O+uydYck1hogppHDW3+Y+3k= +github.com/openimsdk/protocol v0.0.69-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 9ba22f30aeaffb5d5961b3cc8cf8b0304305b925 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 11:19:32 +0800 Subject: [PATCH 025/147] friend incr sync --- cmd/openim-rpc/openim-rpc-friend/main.go | 4 + go.mod | 6 +- internal/rpc/friend/friend.go | 3 + internal/rpc/friend/sync.go | 50 ++++++++++- internal/rpc/user/user.go | 108 ++++++++++++++++------- pkg/common/cmd/msg_gateway_test.go | 7 ++ pkg/common/convert/user.go | 28 +++--- pkg/common/db/cache/friend.go | 13 +++ pkg/common/db/controller/friend.go | 21 +++++ pkg/common/db/dataver/common.go | 13 +-- pkg/common/db/mgo/friend.go | 39 +++++++- pkg/common/db/table/relation/friend.go | 6 ++ 12 files changed, 239 insertions(+), 59 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-friend/main.go b/cmd/openim-rpc/openim-rpc-friend/main.go index 745c405532..4589fb30d5 100644 --- a/cmd/openim-rpc/openim-rpc-friend/main.go +++ b/cmd/openim-rpc/openim-rpc-friend/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewFriendRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/go.mod b/go.mod index 0b2f3cb3b1..0ae542cd2f 100644 --- a/go.mod +++ b/go.mod @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +replace ( + github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index b49490f264..31965adb98 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -62,6 +62,9 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { + if config.RpcConfig.FriendSyncCount < 1 { + config.RpcConfig.FriendSyncCount = constant.MaxSyncPullNumber + } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a2f49b283f..dd6e49250c 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -8,11 +8,55 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/tools/errs" ) +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { + if req.NewUserInfo == nil { + var err error + req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) + if err != nil { + return nil, err + } + } + if req.UserID != req.NewUserInfo.UserID { + return nil, errs.ErrArgs.WrapMsg("req.UserID != req.NewUserInfo.UserID") + } + userIDs, err := s.friendDatabase.FindFriendUserID(ctx, req.UserID) + if err != nil { + return nil, err + } + if len(userIDs) > 0 { + if err := s.friendDatabase.UpdateFriendUserInfo(ctx, req.UserID, userIDs, req.NewUserInfo.Nickname, req.NewUserInfo.FaceURL); err != nil { + return nil, err + } + s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) + } + return &pbfriend.NotificationUserInfoUpdateResp{}, nil +} + func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { - //TODO implement me - panic("implement me") + if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { + return nil, err + } + if req.Keyword == "" { + total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + if err != nil { + return nil, err + } + return &pbfriend.SearchFriendsResp{ + Total: total, + Friends: friendsDB2PB(friends), + }, nil + } + total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) + if err != nil { + return nil, err + } + return &pbfriend.SearchFriendsResp{ + Total: total, + Friends: friendsDB2PB(friends), + }, nil } func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { @@ -54,7 +98,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(incrVer.Version), - VersionID: incrVer.ID.String(), + VersionID: incrVer.ID.Hex(), Full: incrVer.Full(), SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), DeleteUserIds: deleteUserIDs, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 2296ee62fa..92f897ea3f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,12 +16,16 @@ package user import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + friendpb "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/db/redisutil" "math/rand" "strings" + "sync" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -56,11 +60,6 @@ type userServer struct { webhookClient *webhook.Client } -func (s *userServer) SearchUser(ctx context.Context, req *pbuser.SearchUserReq) (*pbuser.SearchUserResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.User RedisConfig config.Redis @@ -136,26 +135,29 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -170,24 +172,28 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse return nil, err } data := convert.UserPb2DBMapEx(req.UserInfo) - if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -688,3 +694,41 @@ func (s *userServer) userModelToResp(users []*relation.UserModel, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } + +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *relation.UserModel) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if *user == *oldUser { + return nil + } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, userID) + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} diff --git a/pkg/common/cmd/msg_gateway_test.go b/pkg/common/cmd/msg_gateway_test.go index d820627b50..2b68a3e3ab 100644 --- a/pkg/common/cmd/msg_gateway_test.go +++ b/pkg/common/cmd/msg_gateway_test.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/utils/jsonutil" "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson/primitive" "math" "testing" ) @@ -59,3 +60,9 @@ func TestName(t *testing.T) { t.Logf("%+v\n", rReso) } + +func TestName1(t *testing.T) { + t.Log(primitive.NewObjectID().String()) + t.Log(primitive.NewObjectID().Hex()) + +} diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index a9378e1a0f..c7c90edb3f 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -15,27 +15,27 @@ package convert import ( + "github.com/openimsdk/tools/utils/datautil" "time" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/protocol/sdkws" ) -func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { - result := make([]*sdkws.UserInfo, 0, len(users)) - for _, user := range users { - userPb := &sdkws.UserInfo{ - UserID: user.UserID, - Nickname: user.Nickname, - FaceURL: user.FaceURL, - Ex: user.Ex, - CreateTime: user.CreateTime.UnixMilli(), - AppMangerLevel: user.AppMangerLevel, - GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, - } - result = append(result, userPb) +func UserDB2Pb(user *relationtb.UserModel) *sdkws.UserInfo { + return &sdkws.UserInfo{ + UserID: user.UserID, + Nickname: user.Nickname, + FaceURL: user.FaceURL, + Ex: user.Ex, + CreateTime: user.CreateTime.UnixMilli(), + AppMangerLevel: user.AppMangerLevel, + GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, } - return result +} + +func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { + return datautil.Slice(users, UserDB2Pb) } func UserPb2DB(user *sdkws.UserInfo) *relationtb.UserModel { diff --git a/pkg/common/db/cache/friend.go b/pkg/common/db/cache/friend.go index 4a51a8c178..eb984d8a45 100644 --- a/pkg/common/db/cache/friend.go +++ b/pkg/common/db/cache/friend.go @@ -52,6 +52,8 @@ type FriendCache interface { // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) @@ -196,6 +198,17 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) FriendCache { + newFriendCache := f.NewCache() + + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendKey(ownerUserID, friendUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) diff --git a/pkg/common/db/controller/friend.go b/pkg/common/db/controller/friend.go index 49c0cb990e..0eef2bee3d 100644 --- a/pkg/common/db/controller/friend.go +++ b/pkg/common/db/controller/friend.go @@ -80,6 +80,12 @@ type FriendDatabase interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error + + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) } type friendDatabase struct { @@ -357,3 +363,18 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } + +func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + return f.friend.FindFriendUserID(ctx, friendUserID) +} + +func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { + if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { + return err + } + return f.cache.DelOwner(friendUserID, ownerUserIDs).ExecDel(ctx) +} + +func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { + return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +} diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index e29ab2b967..5ade1fd582 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -26,13 +26,11 @@ type WriteLog struct { Deleted uint `bson:"deleted"` LastUpdate time.Time `bson:"last_update"` LogLen int `bson:"log_len"` + queryDoc bool `bson:"-"` } func (w *WriteLog) Full() bool { - if w.Version == 0 { - return true - } - return len(w.Logs) != w.LogLen + return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen } func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { @@ -211,7 +209,12 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, } func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - return mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err != nil { + return nil, err + } + res.queryDoc = true + return res, nil } func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 37ba2d7ac2..cf84acb1cf 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -145,13 +145,20 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}) + opt := options.Find().SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}) return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}).SetLimit(int64(limit)) + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) + //res, err := mongoutil.Find[string](ctx, f.coll, filter, opt) + //if err != nil { + // errMsg := err.Error() + // _ = errMsg + // return nil, err + //} + //return res, nil return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -193,6 +200,34 @@ func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, ver return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } +func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + filter := bson.M{ + "friend_user_id": friendUserID, + } + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) +} + +func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { + filter := bson.M{ + "friend_user_id": friendUserID, + } + _, err := mongoutil.UpdateMany(ctx, f.coll, filter, bson.M{"$set": bson.M{"nickname": nickname, "face_url": faceURL}}) + return err +} + +func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { + //where := bson.M{ + // "owner_user_id": ownerUserID, + // "$or": []bson.M{ + // {"remark": bson.M{"$regex": keyword, "$options": "i"}}, + // {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, + // {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, + // }, + //} + //return f.aggregatePagination(ctx, where, pagination) + panic("todo") +} + func IncrVersion(dbs ...func() error) error { for _, fn := range dbs { if err := fn(); err != nil { diff --git a/pkg/common/db/table/relation/friend.go b/pkg/common/db/table/relation/friend.go index a28b2278d3..4970a04206 100644 --- a/pkg/common/db/table/relation/friend.go +++ b/pkg/common/db/table/relation/friend.go @@ -66,4 +66,10 @@ type FriendModelInterface interface { UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error + + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*FriendModel, error) } From eb362daaf28e37502f1bd595a2c4cfd30bff5a33 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:07:42 +0800 Subject: [PATCH 026/147] mage --- internal/rpc/friend/convert.go | 6 ++-- internal/rpc/friend/sync.go | 4 +-- internal/rpc/user/user.go | 2 +- pkg/common/convert/user.go | 2 +- pkg/common/storage/cache/friend.go | 9 ++++++ pkg/common/storage/cache/redis/friend.go | 31 ++----------------- pkg/common/storage/controller/friend.go | 8 ++--- pkg/common/storage/database/friend.go | 19 ++---------- pkg/common/storage/database/mgo/friend.go | 2 +- pkg/common/storage/database/mgo/mongo_test.go | 0 pkg/common/storage/model/friend.go | 2 ++ 11 files changed, 29 insertions(+), 56 deletions(-) delete mode 100644 pkg/common/storage/database/mgo/mongo_test.go diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go index aa68952843..0730df5291 100644 --- a/internal/rpc/friend/convert.go +++ b/internal/rpc/friend/convert.go @@ -1,12 +1,12 @@ package friend import ( - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/friend" "github.com/openimsdk/tools/utils/datautil" ) -func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { +func friendDB2PB(db *model.Friend) *friend.FriendInfo { return &friend.FriendInfo{ OwnerUserID: db.OwnerUserID, FriendUserID: db.FriendUserID, @@ -21,6 +21,6 @@ func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { } } -func friendsDB2PB(db []*relationtb.FriendModel) []*friend.FriendInfo { +func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { return datautil.Slice(db, friendDB2PB) } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index dd6e49250c..bfa7e3a574 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -6,7 +6,7 @@ import ( "encoding/binary" "encoding/json" "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbfriend "github.com/openimsdk/protocol/friend" "github.com/openimsdk/tools/errs" ) @@ -89,7 +89,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } else { deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() } - var friends []*relation.FriendModel + var friends []*model.Friend if len(changeUserIDs) > 0 { friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) if err != nil { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 14ce09ed55..504c08a2ca 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -695,7 +695,7 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } -func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *relation.UserModel) error { +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { user, err := s.db.GetUserByID(ctx, userID) if err != nil { return err diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index 5595e8fe52..d824fa68e0 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -34,7 +34,7 @@ func UserDB2Pb(user *relationtb.User) *sdkws.UserInfo { } } -func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { +func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { return datautil.Slice(users, UserDB2Pb) } diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index acff829f86..9dec3d5ab4 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -16,6 +16,7 @@ package cache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -32,4 +33,12 @@ type FriendCache interface { DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + + DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 7b93c74aca..d6754d7bf9 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -28,37 +28,12 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( friendExpireTime = time.Second * 60 * 60 * 12 ) -//// FriendCache is an interface for caching friend-related data. -//type FriendCache interface { -// metaCache -// NewCache() FriendCache -// GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) -// // Called when friendID list changed -// DelFriendIDs(ownerUserID ...string) FriendCache -// -// DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache -// -// // Get single friendInfo from the cache -// GetFriend(ctx context.Context, ownerUserID, friendUserID string) (friend *relationtb.FriendModel, err error) -// // Delete friend when friend info changed -// DelFriend(ownerUserID, friendUserID string) FriendCache -// // Delete friends when friends' info changed -// DelFriends(ownerUserID string, friendUserIDs []string) FriendCache -// -// DelOwner(friendUserID string, ownerUserIDs []string) FriendCache -// -// FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) -// -// FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) -//} - // FriendCacheRedis is an implementation of the FriendCache interface using Redis. type FriendCacheRedis struct { cache.BatchDeleter @@ -129,8 +104,8 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache { - newGroupCache := f.NewCache() +func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { + newGroupCache := f.CloneFriendCache() keys := make([]string, 0, len(ownerUserIDs)) for _, userID := range ownerUserIDs { keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) @@ -194,7 +169,7 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } -func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) FriendCache { +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) cache.FriendCache { newFriendCache := f.CloneFriendCache() for _, ownerUserID := range ownerUserIDs { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 9b177dc45c..0c83930b1d 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -87,7 +87,7 @@ type FriendDatabase interface { UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) } type friendDatabase struct { @@ -289,7 +289,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -374,9 +374,9 @@ func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { return err } - return f.cache.DelOwner(friendUserID, ownerUserIDs).ExecDel(ctx) + return f.cache.DelOwner(friendUserID, ownerUserIDs).ChainExecDel(ctx) } -func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { +func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) } diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 728e195935..f5a999485c 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -17,28 +17,13 @@ package database import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) -//// FriendModel represents the data structure for a friend relationship in MongoDB. -//type FriendModel struct { -// OwnerUserID string `bson:"owner_user_id"` -// FriendUserID string `bson:"friend_user_id"` -// FriendNickname string `bson:"friend_nickname"` -// FriendFaceURL string `bson:"friend_face_url"` -// Remark string `bson:"remark"` -// CreateTime time.Time `bson:"create_time"` -// AddSource int32 `bson:"add_source"` -// OperatorUserID string `bson:"operator_user_id"` -// Ex string `bson:"ex"` -// IsPinned bool `bson:"is_pinned"` -//} - // Friend defines the operations for managing friends in MongoDB. -type FriendModelInterface interface { +type Friend interface { // Create inserts multiple friend records. Create(ctx context.Context, friends []*model.Friend) (err error) // Delete removes specified friends of the owner user. @@ -71,4 +56,6 @@ type FriendModelInterface interface { UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index dae5f21034..8993ee8293 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -225,7 +225,7 @@ func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID strin return err } -func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { +func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { //where := bson.M{ // "owner_user_id": ownerUserID, // "$or": []bson.M{ diff --git a/pkg/common/storage/database/mgo/mongo_test.go b/pkg/common/storage/database/mgo/mongo_test.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2a..c7675e466d 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -22,6 +22,8 @@ import ( type Friend struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` + FriendNickname string `bson:"friend_nickname"` + FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` From 0aaf8b93a020f5c90e4f17019ebb1bc09e76c58c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:41:45 +0800 Subject: [PATCH 027/147] optimization version log --- go.mod | 8 +- go.sum | 6 +- pkg/common/db/dataver/result.go | 42 --- pkg/common/db/dataver/todo.go | 38 --- pkg/common/listdemo/common.go | 182 ---------- pkg/common/listdemo/friend_model.go | 5 - pkg/common/listdemo/friend_table.go | 86 ----- pkg/common/listdemo2/common.go | 322 ------------------ pkg/common/listdemo2/common_test.go | 61 ---- pkg/common/listdemo2/demo.js | 86 ----- pkg/common/listdemo2/demo2.js | 63 ---- pkg/common/listdemo2/demo3.js | 59 ---- pkg/common/listdemo2/demo5.js | 10 - pkg/common/storage/cache/friend.go | 3 +- pkg/common/storage/cache/redis/friend.go | 3 +- pkg/common/storage/controller/friend.go | 5 +- pkg/common/storage/database/friend.go | 3 +- pkg/common/storage/database/mgo/friend.go | 16 +- .../database/mgo/version_log.go} | 116 ++----- pkg/common/storage/database/version_log.go | 18 + pkg/common/storage/model/user.go | 4 +- pkg/common/storage/model/version_log.go | 61 ++++ 22 files changed, 120 insertions(+), 1077 deletions(-) delete mode 100644 pkg/common/db/dataver/result.go delete mode 100644 pkg/common/db/dataver/todo.go delete mode 100644 pkg/common/listdemo/common.go delete mode 100644 pkg/common/listdemo/friend_model.go delete mode 100644 pkg/common/listdemo/friend_table.go delete mode 100644 pkg/common/listdemo2/common.go delete mode 100644 pkg/common/listdemo2/common_test.go delete mode 100644 pkg/common/listdemo2/demo.js delete mode 100644 pkg/common/listdemo2/demo2.js delete mode 100644 pkg/common/listdemo2/demo3.js delete mode 100644 pkg/common/listdemo2/demo5.js rename pkg/common/{db/dataver/common.go => storage/database/mgo/version_log.go} (56%) create mode 100644 pkg/common/storage/database/version_log.go create mode 100644 pkg/common/storage/model/version_log.go diff --git a/go.mod b/go.mod index 0ae542cd2f..bd1e16a5b6 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.2 - github.com/openimsdk/tools v0.0.49-alpha.23 - github.com/pkg/errors v0.9.1 + github.com/openimsdk/tools v0.0.49-alpha.24 + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 @@ -178,6 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace ( - github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -) +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index c0bd78645e..1cc2cc1bc0 100644 --- a/go.sum +++ b/go.sum @@ -286,10 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.2 h1:XNc3pmAXyW+PMo7tghr2O+uydYck1hogppHDW3+Y+3k= -github.com/openimsdk/protocol v0.0.69-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= -github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= +github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go deleted file mode 100644 index ec09ffa314..0000000000 --- a/pkg/common/db/dataver/result.go +++ /dev/null @@ -1,42 +0,0 @@ -package dataver - -import ( - "github.com/openimsdk/tools/utils/datautil" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SyncResult struct { - Version uint - VersionID string - DeleteEID []string - Changes []string - Full bool -} - -func VersionIDStr(id primitive.ObjectID) string { - if id.IsZero() { - return "" - } - return id.String() -} - -func NewSyncResult(wl *WriteLog, fullIDs []string, versionID string) *SyncResult { - var findEIDs []string - var res SyncResult - if wl.Full() || VersionIDStr(wl.ID) != versionID { - res.Changes = fullIDs - res.Full = true - } else { - idSet := datautil.SliceSet(fullIDs) - for _, l := range wl.Logs { - if l.Deleted { - res.DeleteEID = append(res.DeleteEID, l.EID) - } else { - if _, ok := idSet[l.EID]; ok { - findEIDs = append(findEIDs, l.EID) - } - } - } - } - return &res -} diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go deleted file mode 100644 index cd2be14f01..0000000000 --- a/pkg/common/db/dataver/todo.go +++ /dev/null @@ -1,38 +0,0 @@ -package dataver - -/* - -UserIDs 顺序 -前500顺序 - - -1,2,3,4,5,6,7,8,9 - -1,3,5,7,8,9 - - -1.sdk添加一个表记录 docID(后续换名字), version -2.sdk同步，先计算idHash，api调用参数idHash, docID, version -3.服务器先判断version变更记录，没有直接返回同步成功。 - 有变更，先查版本变更记录，在查前500id，变更记录只保留前500id中的 - 根据前500id计算idHash，不一致返回会全量id，不反悔删除id - 全量同步有标识，只返回全量id - 变更记录只包含id，不包括详细信息。 -4.sdk通过变更记录，同步数据不一致进行重试。 -5.先修改db，在自增版本号，外层加事务 - - - - - - - - - - - - - - - -*/ diff --git a/pkg/common/listdemo/common.go b/pkg/common/listdemo/common.go deleted file mode 100644 index 2d99cd8c9f..0000000000 --- a/pkg/common/listdemo/common.go +++ /dev/null @@ -1,182 +0,0 @@ -package listdemo - -import ( - "context" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" - "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -var ( - ErrListNotFound = errors.New("list not found") - ErrElemExist = errors.New("elem exist") - ErrNotFound = mongo.ErrNoDocuments -) - -type ListDoc interface { - IDName() string // 外层业务id字段名字 user_id - ElemsName() string // 外层列表名字 friends - VersionName() string // 外层版本号 version - DeleteVersion() string // 删除版本号 - BuildDoc(lid any, e Elem) any // 返回一个组装的doc文档 -} - -type Elem interface { - IDName() string // 业务id名字 friend_user_id - IDValue() any // 业务id值 userID -> "100000000" - VersionName() string // 版本号 - DeletedName() string // 删除字段名字 - ToMap() map[string]any // 把结构体转换为map -} - -type List[D any, E Elem] struct { - coll *mongo.Collection - lf ListDoc -} - -func (l *List[D, E]) zeroE() E { - var t E - return t -} - -func (l *List[D, E]) FindElem(ctx context.Context, lid any, eid any) (E, error) { - res, err := l.FindElems(ctx, lid, []any{eid}) - if err != nil { - return l.zeroE(), err - } - if len(res) == 0 { - return l.zeroE(), ErrNotFound - } - return res[0], nil -} - -// FindElems 查询Elems -func (l *List[D, E]) FindElems(ctx context.Context, lid any, eids []any) ([]E, error) { - //pipeline := []bson.M{ - // { - // "$match": bson.M{ - // l.lf.IDName(): lid, - // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ - // "$in": eids, - // }, - // }, - // }, - // { - // "$unwind": "$" + l.lf.ElemsName(), - // }, - // { - // "$match": bson.M{ - // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ - // "$in": eids, - // }, - // }, - // }, - //} - panic("todo") -} - -func (l *List[D, E]) Find(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { - return nil, nil -} - -func (l *List[D, E]) Count(ctx context.Context, filter any, opts ...*options.CountOptions) (int64, error) { - return 0, nil -} - -func (l *List[D, E]) Update(ctx context.Context, lid any, eid any) (*mongo.UpdateResult, error) { - - return nil, nil -} - -func (l *List[D, E]) Delete(ctx context.Context, lid any, eids any) (*mongo.UpdateResult, error) { - - return nil, nil -} - -func (l *List[D, E]) Page(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []E, error) { - return 0, nil, nil -} - -func (l *List[D, E]) ElemIDs(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { - - return nil, nil -} - -// InsertElem 插入一个 -func (l *List[D, E]) InsertElem(ctx context.Context, lid any, e Elem) error { - if err := l.insertElem(ctx, lid, e); err == nil { - return nil - } else if !errors.Is(err, ErrListNotFound) { - return err - } - if _, err := l.coll.InsertOne(ctx, l.lf.BuildDoc(lid, e)); err == nil { - return nil - } else if mongo.IsDuplicateKeyError(err) { - return l.insertElem(ctx, lid, e) - } else { - return err - } -} - -func (l *List[D, E]) insertElem(ctx context.Context, lid any, e Elem) error { - data := e.ToMap() - data[e.VersionName()] = "$max_version" - filter := bson.M{ - l.lf.IDName(): lid, - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "found_elem": bson.M{ - "$in": bson.A{e.IDValue(), l.lf.ElemsName() + "." + e.IDName()}, - }, - }, - }, - { - "$set": bson.M{ - "max_version": bson.M{ - "$cond": bson.M{ - "if": "$found_elem", - "then": "$max_version", - "else": bson.M{"$add": bson.A{"max_version", 1}}, - }, - }, - }, - }, - { - "$set": bson.M{ - l.lf.ElemsName(): bson.M{ - "$cond": bson.M{ - "if": "$found_elem", - "then": "$" + l.lf.ElemsName(), - "else": bson.M{ - "$concatArrays": bson.A{ - "$" + l.lf.ElemsName(), - bson.A{ - data, - }, - }, - }, - }, - }, - }, - }, - { - "$unset": "found_elem", - }, - } - res, err := mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) - if err != nil { - return err - } - if res.MatchedCount == 0 { - return ErrListNotFound - } - if res.ModifiedCount == 0 { - return ErrElemExist - } - return nil -} diff --git a/pkg/common/listdemo/friend_model.go b/pkg/common/listdemo/friend_model.go deleted file mode 100644 index 13507bfa3f..0000000000 --- a/pkg/common/listdemo/friend_model.go +++ /dev/null @@ -1,5 +0,0 @@ -package listdemo - -type friendModel struct { - db *List[*Friend, *FriendElem] -} diff --git a/pkg/common/listdemo/friend_table.go b/pkg/common/listdemo/friend_table.go deleted file mode 100644 index b1ad877a21..0000000000 --- a/pkg/common/listdemo/friend_table.go +++ /dev/null @@ -1,86 +0,0 @@ -package listdemo - -import ( - "time" -) - -var ( - _ Elem = (*FriendElem)(nil) - _ ListDoc = (*Friend)(nil) -) - -type FriendElem struct { - FriendUserID string `bson:"friend_user_id"` - Nickname string `bson:"nickname"` - FaceURL string `bson:"face_url"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` - Version uint `bson:"version"` - DeleteTime *time.Time `bson:"delete_time"` -} - -func (f *FriendElem) IDName() string { - return "friend_user_id" -} - -func (f *FriendElem) IDValue() any { - return f.FriendUserID -} - -func (f *FriendElem) VersionName() string { - return "version" -} - -func (f *FriendElem) DeletedName() string { - return "delete_time" -} - -func (f *FriendElem) ToMap() map[string]any { - return map[string]any{ - "friend_user_id": f.FriendUserID, - "nickname": f.Nickname, - "face_url": f.FaceURL, - "remark": f.Remark, - "create_time": f.CreateTime, - "add_source": f.AddSource, - "operator_user_id": f.OperatorUserID, - "ex": f.Ex, - "is_pinned": f.IsPinned, - "version": f.Version, - "delete_time": f.DeleteTime, - } -} - -type Friend struct { - UserID string `bson:"user_id"` - Friends []*FriendElem `bson:"friends"` - Version uint `bson:"version"` - DeleteVersion uint `bson:"delete_version"` -} - -func (f *Friend) BuildDoc(lid any, e Elem) any { - return &Friend{ - UserID: lid.(string), - Friends: []*FriendElem{e.(*FriendElem)}, - } -} - -func (f *Friend) ElemsID() string { - return "user_id" -} - -func (f *Friend) IDName() string { - return "user_id" -} - -func (f *Friend) ElemsName() string { - return "friends" -} - -func (f *Friend) VersionName() string { - return "version" -} diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go deleted file mode 100644 index 40419d533c..0000000000 --- a/pkg/common/listdemo2/common.go +++ /dev/null @@ -1,322 +0,0 @@ -package listdemo - -import ( - "context" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" - "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "time" -) - -var ( - ErrListNotFound = errors.New("list not found") - ErrElemExist = errors.New("elem exist") - ErrNeedFull = errors.New("need full") - ErrNotFound = mongo.ErrNoDocuments -) - -const ( - FirstVersion = 1 - DefaultDeleteVersion = 0 -) - -type Elem struct { - ID string - Version uint -} - -type ChangeLog struct { - ChangeIDs []Elem - DeleteIDs []Elem -} - -type WriteLog struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` -} - -type WriteLogLen struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` -} - -type LogElem struct { - EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` -} - -type LogModel struct { - coll *mongo.Collection -} - -func (l *LogModel) InitIndex(ctx context.Context) error { - _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.M{ - "d_id": 1, - }, - }) - return err -} - -func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - filter := bson.M{ - "d_id": dId, - } - elem := bson.M{ - "e_id": eId, - "version": "$version", - "deleted": deleted, - "last_update": now, - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "elem_index": bson.M{ - "$indexOfArray": []any{"$logs.e_id", eId}, - }, - }, - }, - { - "$set": bson.M{ - "version": bson.M{"$add": []any{"$version", 1}}, - "last_update": now, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$lt": []any{"$elem_index", 0}, - }, - "then": bson.M{ - "$concatArrays": []any{ - "$logs", - []bson.M{ - elem, - }, - }, - }, - "else": bson.M{ - "$map": bson.M{ - "input": bson.M{ - "$range": []any{0, bson.M{"$size": "$logs"}}, - }, - "as": "i", - "in": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$eq": []any{"$$i", "$elem_index"}, - }, - "then": elem, - "else": bson.M{ - "$arrayElemAt": []any{ - "$logs", - "$$i", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - "$unset": "elem_index", - }, - } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) -} - -func (l *LogModel) WriteLogBatch(ctx context.Context, dId string, eIds []string, deleted bool) error { - if len(eIds) == 0 { - return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) - } - if datautil.Duplicate(eIds) { - return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) - } - now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil - } - wl := WriteLog{ - DID: dId, - Logs: make([]LogElem, 0, len(eIds)), - Version: FirstVersion, - Deleted: DefaultDeleteVersion, - LastUpdate: now, - } - for _, eId := range eIds { - wl.Logs = append(wl.Logs, LogElem{ - EID: eId, - Deleted: deleted, - Version: FirstVersion, - LastUpdate: now, - }) - } - if _, err := l.coll.InsertOne(ctx, &wl); err == nil { - return nil - } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { - return err - } else if res.ModifiedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) - } - return nil -} - -func (l *LogModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - if len(eIds) == 0 { - return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) - } - filter := bson.M{ - "d_id": dId, - } - elems := make([]bson.M, 0, len(eIds)) - for _, eId := range eIds { - elems = append(elems, bson.M{ - "e_id": eId, - "version": "$version", - "deleted": deleted, - "last_update": now, - }) - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "delete_e_ids": eIds, - }, - }, - { - "$set": bson.M{ - "version": bson.M{"$add": []any{"$version", 1}}, - "last_update": now, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$filter": bson.M{ - "input": "$logs", - "as": "log", - "cond": bson.M{ - "$not": bson.M{ - "$in": []any{"$$log.e_id", "$delete_e_ids"}, - }, - }, - }, - }, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$concatArrays": []any{ - "$logs", - elems, - }, - }, - }, - }, - { - "$unset": "delete_e_ids", - }, - } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) -} - -func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLogLen, error) { - pipeline := []bson.M{ - { - "$match": bson.M{ - "d_id": did, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$or": []bson.M{ - {"$lt": []any{"$version", version}}, - {"$gte": []any{"$deleted", version}}, - }, - }, - "then": []any{}, - "else": "$logs", - }, - }, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$filter": bson.M{ - "input": "$logs", - "as": "l", - "cond": bson.M{ - "$gt": []any{"$$l.version", version}, - }, - }, - }, - }, - }, - { - "$addFields": bson.M{ - "log_len": bson.M{"$size": "$logs"}, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$gt": []any{"$log_len", limit}, - }, - "then": []any{}, - "else": "$logs", - }, - }, - }, - }, - } - if limit <= 0 { - pipeline = pipeline[:len(pipeline)-1] - } - res, err := mongoutil.Aggregate[*WriteLogLen](ctx, l.coll, pipeline) - if err != nil { - return nil, err - } - if len(res) == 0 { - return &WriteLogLen{}, nil - } - return res[0], nil -} - -func (l *LogModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { - return mongoutil.DeleteMany(ctx, l.coll, bson.M{ - "last_update": bson.M{ - "$lt": deadline, - }, - }) -} diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go deleted file mode 100644 index c03cc31bd2..0000000000 --- a/pkg/common/listdemo2/common_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package listdemo - -import ( - "context" - "errors" - "github.com/openimsdk/tools/db/mongoutil" - "testing" -) - -func Result[V any](val V, err error) V { - if err != nil { - panic(err) - } - return val -} - -func Check(err error) { - if err != nil { - panic(err) - } -} - -func TestName(t *testing.T) { - cli := Result(mongoutil.NewMongoDB(context.Background(), &mongoutil.Config{Uri: "mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100", Database: "openim_v3"})) - - db := cli.GetDB() - tx := cli.GetTx() - - const num = 1 - lm := &LogModel{coll: db.Collection("friend_version")} - - err := tx.Transaction(context.Background(), func(ctx context.Context) error { - err := tx.Transaction(ctx, func(ctx context.Context) error { - return lm.WriteLogBatch(ctx, "100", []string{"1000", "2000"}, true) - }) - if err != nil { - t.Log("--------->") - return err - } - return errors.New("1234") - }) - t.Log(err) - - //start := time.Now() - //eIds := make([]string, 0, num) - //for i := 0; i < num; i++ { - // eIds = append(eIds, strconv.Itoa(1000+(i))) - //} - //lm.WriteLogBatch1(context.Background(), "100", eIds, false) - //end := time.Now() - //t.Log(end.Sub(start)) // 509.962208ms - //t.Log(end.Sub(start) / num) // 511.496µs - - //start := time.Now() - //wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) - //if err != nil { - // panic(err) - //} - //t.Log(time.Since(start)) - //t.Log(wll) -} diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js deleted file mode 100644 index be9c74cb98..0000000000 --- a/pkg/common/listdemo2/demo.js +++ /dev/null @@ -1,86 +0,0 @@ -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - { - $addFields: { - elem_index: { - $indexOfArray: [ - "$logs.e_id", - "1000" - ] - } - } - }, - { - $set: { - version: { - $add: ["$version", 1] - }, - last_update: new Date(), - - } - }, - { - $set: { - logs: { - $cond: { - if: { - $lt: ["$elem_index", 0] - }, - then: { - $concatArrays: [ - "$logs", - [ - { - e_id: "1000", - last_update: new Date(), - version: "$version", - deleted: false - } - ] - ] - }, - else: { - $map: { - input: { - $range: [0, { - $size: "$logs" - }] - }, - as: "i", - in: { - $cond: { - if: { - $eq: ["$$i", "$elem_index"] - }, - then: { - e_id: "1000", - last_update: new Date(), - version: "$version", - deleted: false - }, - else: { - $arrayElemAt: ["$logs", "$$i"] - } - }, - - }, - - }, - - }, - - }, - - }, - - }, - - }, - { - $unset: ["elem_index"] - }, - ] -) diff --git a/pkg/common/listdemo2/demo2.js b/pkg/common/listdemo2/demo2.js deleted file mode 100644 index 15e7abcfe0..0000000000 --- a/pkg/common/listdemo2/demo2.js +++ /dev/null @@ -1,63 +0,0 @@ - -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - { - $addFields: { - update_elem_ids: ["1000", "1001","1003", "2000"] - } - }, - { - $set: { - version: { - $add: ["$version", 1] - }, - last_update: new Date(), - } - }, - { - $set: { - logs: { - $filter: { - input: "$logs", - as: "log", - cond: { - "$not": { - $in: ["$$log.e_id", "$update_elem_ids"] - } - } - } - }, - - }, - - }, - { - $set: { - logs: { - $concatArrays: [ - "$logs", - [ - { - e_id: "1003", - last_update: ISODate("2024-05-25T06:32:10.238Z"), - version: "$version", - deleted: false - }, - - ] - ] - } - } - }, - { - $unset: ["update_elem_ids"] - }, - - ] -) - - - diff --git a/pkg/common/listdemo2/demo3.js b/pkg/common/listdemo2/demo3.js deleted file mode 100644 index 5367971cf9..0000000000 --- a/pkg/common/listdemo2/demo3.js +++ /dev/null @@ -1,59 +0,0 @@ - -db.friend_version.aggregate([ - { - "$match": { - "d_id": "100", - } - }, - { - "$project": { - "_id": 0, - "d_id": 0, - } - }, - { - "$addFields": { - "logs": { - $cond: { - if: { - $or: [ - {$lt: ["$version", 3]}, - {$gte: ["$deleted", 3]}, - ], - }, - then: [], - else: "$logs", - } - } - }, - }, - { - "$addFields": { - "logs": { - "$filter": { - input: "$logs", - as: "l", - cond: { $gt: ["$$l.version", 3] } - } - } - } - }, - { - "$addFields": { - "log_len": { - $size: "$logs" - } - } - }, - { - "$addFields": { - "logs": { - $cond: { - if: {$gt: ["$log_len", 1]}, - then: [], - else: "$logs", - } - } - } - } -]) diff --git a/pkg/common/listdemo2/demo5.js b/pkg/common/listdemo2/demo5.js deleted file mode 100644 index 71a709a811..0000000000 --- a/pkg/common/listdemo2/demo5.js +++ /dev/null @@ -1,10 +0,0 @@ -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - - - - ], -) \ No newline at end of file diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 9dec3d5ab4..3fee297ac8 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -16,7 +16,6 @@ package cache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -40,5 +39,5 @@ type FriendCache interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index d6754d7bf9..f4edbca9a3 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -16,7 +16,6 @@ package redis import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/dtm-labs/rockscache" @@ -186,6 +185,6 @@ func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserI }) } -func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 0c83930b1d..1af967b9bb 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -17,7 +17,6 @@ package controller import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -81,7 +80,7 @@ type FriendDatabase interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) @@ -362,7 +361,7 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) } -func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index f5a999485c..6ab1185bc0 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -16,7 +16,6 @@ package database import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" @@ -49,7 +48,7 @@ type Friend interface { // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) - FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 8993ee8293..19b2f4f184 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -16,7 +16,6 @@ package mgo import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -30,7 +29,7 @@ import ( // FriendMgo implements Friend using MongoDB as the storage backend. type FriendMgo struct { coll *mongo.Collection - owner dataver.DataLog + owner database.VersionLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -46,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := dataver.NewDataLog(db.Collection("friend_owner_log")) + owner, err := NewVersionLog(db.Collection("friend_owner_version_log")) if err != nil { return nil, err } @@ -100,15 +99,6 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) @@ -206,7 +196,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien }) } -func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/storage/database/mgo/version_log.go similarity index 56% rename from pkg/common/db/dataver/common.go rename to pkg/common/storage/database/mgo/version_log.go index 5ade1fd582..fe123073e9 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -1,8 +1,10 @@ -package dataver +package mgo import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" @@ -13,84 +15,19 @@ import ( "time" ) -const ( - FirstVersion = 1 - DefaultDeleteVersion = 0 -) - -type WriteLog struct { - ID primitive.ObjectID `bson:"_id"` - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` - queryDoc bool `bson:"-"` -} - -func (w *WriteLog) Full() bool { - return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen -} - -func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { - for _, l := range w.Logs { - if l.Deleted { - delIds = append(delIds, l.EID) - } else { - changeIds = append(changeIds, l.EID) - } - } - return -} - -type Elem struct { - EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` -} - -type tableWriteLog struct { - ID primitive.ObjectID `bson:"_id"` - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` -} - -func (t *tableWriteLog) WriteLog() *WriteLog { - return &WriteLog{ - ID: t.ID, - DID: t.DID, - Logs: t.Logs, - Version: t.Version, - Deleted: t.Deleted, - LastUpdate: t.LastUpdate, - LogLen: 0, - } -} - -type DataLog interface { - WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error - FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) - DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error -} - -func NewDataLog(coll *mongo.Collection) (DataLog, error) { - lm := &logModel{coll: coll} +func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { + lm := &VersionLogMgo{coll: coll} if lm.initIndex(context.Background()) != nil { return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) } return lm, nil } -type logModel struct { +type VersionLogMgo struct { coll *mongo.Collection } -func (l *logModel) initIndex(ctx context.Context) error { +func (l *VersionLogMgo) initIndex(ctx context.Context) error { _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ Keys: bson.M{ "d_id": 1, @@ -99,7 +36,7 @@ func (l *logModel) initIndex(ctx context.Context) error { return err } -func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } @@ -127,20 +64,20 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele return nil } -func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*tableWriteLog, error) { - wl := tableWriteLog{ +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*model.VersionLogTable, error) { + wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, - Logs: make([]Elem, 0, len(eIds)), - Version: FirstVersion, - Deleted: DefaultDeleteVersion, + Logs: make([]model.VersionLogElem, 0, len(eIds)), + Version: database.FirstVersion, + Deleted: database.DefaultDeleteVersion, LastUpdate: now, } for _, eId := range eIds { - wl.Logs = append(wl.Logs, Elem{ + wl.Logs = append(wl.Logs, model.VersionLogElem{ EID: eId, Deleted: deleted, - Version: FirstVersion, + Version: database.FirstVersion, LastUpdate: now, }) } @@ -148,7 +85,7 @@ func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, delet return &wl, err } -func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { if eIds == nil { eIds = []string{} } @@ -208,23 +145,22 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) +func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { + vl, err := mongoutil.FindOne[*model.VersionLogTable](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) if err != nil { return nil, err } - res.queryDoc = true - return res, nil + return vl.VersionLog(), nil } -func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { return wl, nil } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { - return res.WriteLog(), nil + return res.VersionLog(), nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) } else { @@ -232,7 +168,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, } } -func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { if version == 0 && limit == 0 { return l.findDoc(ctx, dId) } @@ -293,17 +229,17 @@ func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, if limit <= 0 { pipeline = pipeline[:len(pipeline)-1] } - res, err := mongoutil.Aggregate[*WriteLog](ctx, l.coll, pipeline) + vl, err := mongoutil.Aggregate[*model.VersionLog](ctx, l.coll, pipeline) if err != nil { return nil, err } - if len(res) == 0 { + if len(vl) == 0 { return nil, mongo.ErrNoDocuments } - return res[0], nil + return vl[0], nil } -func (l *logModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { +func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { return mongoutil.DeleteMany(ctx, l.coll, bson.M{ "last_update": bson.M{ "$lt": deadline, diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go new file mode 100644 index 0000000000..6783cf4ff6 --- /dev/null +++ b/pkg/common/storage/database/version_log.go @@ -0,0 +1,18 @@ +package database + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "time" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type VersionLog interface { + WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error +} diff --git a/pkg/common/storage/model/user.go b/pkg/common/storage/model/user.go index c6a4f952c2..f64d09e797 100644 --- a/pkg/common/storage/model/user.go +++ b/pkg/common/storage/model/user.go @@ -36,10 +36,10 @@ func (u *User) GetFaceURL() string { return u.FaceURL } -func (u User) GetUserID() string { +func (u *User) GetUserID() string { return u.UserID } -func (u User) GetEx() string { +func (u *User) GetEx() string { return u.Ex } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go new file mode 100644 index 0000000000..044bd42da4 --- /dev/null +++ b/pkg/common/storage/model/version_log.go @@ -0,0 +1,61 @@ +package model + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type VersionLogElem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type VersionLogTable struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (v *VersionLogTable) VersionLog() *VersionLog { + return &VersionLog{ + ID: v.ID, + DID: v.DID, + Logs: v.Logs, + Version: v.Version, + Deleted: v.Deleted, + LastUpdate: v.LastUpdate, + LogLen: 0, + queryDoc: true, + } +} + +type VersionLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` + queryDoc bool `bson:"-"` +} + +func (w *VersionLog) Full() bool { + return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen +} + +func (w *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { + for _, l := range w.Logs { + if l.Deleted { + delIds = append(delIds, l.EID) + } else { + changeIds = append(changeIds, l.EID) + } + } + return +} From 0f72de85b291c50c85f6638a42e445da1c03813c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:49:34 +0800 Subject: [PATCH 028/147] optimization version log --- pkg/common/storage/database/mgo/friend.go | 24 ++++------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 19b2f4f184..2df85a4302 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -54,7 +54,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { mp := make(map[string][]string) @@ -76,7 +76,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) @@ -92,7 +92,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) @@ -152,13 +152,6 @@ func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pa func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) - //res, err := mongoutil.Find[string](ctx, f.coll, filter, opt) - //if err != nil { - // errMsg := err.Error() - // _ = errMsg - // return nil, err - //} - //return res, nil return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -189,7 +182,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) @@ -227,12 +220,3 @@ func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword strin //return f.aggregatePagination(ctx, where, pagination) panic("todo") } - -func IncrVersion(dbs ...func() error) error { - for _, fn := range dbs { - if err := fn(); err != nil { - return err - } - } - return nil -} From cfc01bb3d962d0445d15a8cd7461cef8ec8d92c9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 31 May 2024 18:41:34 +0800 Subject: [PATCH 029/147] sync --- go.mod | 4 +- go.sum | 2 + internal/api/friend.go | 17 --- internal/api/group.go | 50 ++++++++ internal/api/router.go | 3 + internal/rpc/friend/sync.go | 114 +++++++++++------- internal/rpc/group/convert.go | 4 + internal/rpc/group/group.go | 61 +++++++--- internal/rpc/group/sync.go | 66 +++++++++- internal/rpc/incrversion/option.go | 58 +++++++++ pkg/common/config/config.go | 3 +- pkg/common/storage/cache/group.go | 3 + pkg/common/storage/cache/redis/group.go | 23 ++++ pkg/common/storage/controller/group.go | 55 +++++++-- pkg/common/storage/database/group_member.go | 3 + pkg/common/storage/database/mgo/friend.go | 10 +- .../storage/database/mgo/group_member.go | 95 ++++++++++++--- .../storage/database/mgo/version_log.go | 2 +- pkg/common/storage/database/version_log.go | 2 +- 19 files changed, 462 insertions(+), 113 deletions(-) create mode 100644 internal/rpc/incrversion/option.go diff --git a/go.mod b/go.mod index bd1e16a5b6..db418bf144 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.2 + github.com/openimsdk/protocol v0.0.69-alpha.3 github.com/openimsdk/tools v0.0.49-alpha.24 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -178,4 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 1cc2cc1bc0..4588df18a7 100644 --- a/go.sum +++ b/go.sum @@ -286,6 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= +github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/friend.go b/internal/api/friend.go index 5b2dfa0ca7..3af162a536 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -98,20 +98,3 @@ func (o *FriendApi) UpdateFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) } - -//func BatchIncremental[A, B, C any,D comparable](c *gin.Context, rpc func(client C, ctx context.Context, req *A, options ...grpc.CallOption) (*B, error), getID func(req *A)D, setID func(req *A, id D)) { -// req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq[A]](c) -// if err != nil { -// apiresp.GinError(c, err) -// return -// } -// if len(req.List) == 0 { -// apiresp.GinError(c, errs.ErrArgs.WrapMsg("empty versions list")) -// return -// } -//} -// -//type BatchIncrementalReq[A any] struct { -// UserID string `json:"user_id"` -// List []*A `json:"list"` -//} diff --git a/internal/api/group.go b/internal/api/group.go index 6079c53437..7baa163233 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" + "github.com/openimsdk/tools/apiresp" ) type GroupApi rpcclient.Group @@ -134,3 +135,52 @@ func (o *GroupApi) GetGroups(c *gin.Context) { func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c) } + +func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { + type BatchIncrementalReq[A any] struct { + UserID string `json:"user_id"` + List []*group.GetIncrementalGroupMemberReq `json:"list"` + } + type BatchIncrementalResp struct { + List map[string]*group.GetIncrementalGroupMemberResp `json:"list"` + } + req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq](c) + if err != nil { + apiresp.GinError(c, err) + return + } + resp := &BatchIncrementalResp{ + List: make(map[string]*group.GetIncrementalGroupMemberResp), + } + var ( + changeCount int + ) + for _, req := range req.List { + if _, ok := resp.List[req.GroupID]; ok { + continue + } + res, err := o.Client.GetIncrementalGroupMember(c, req) + if err != nil { + if len(resp.List) == 0 { + apiresp.GinError(c, err) + } else { + apiresp.GinSuccess(c, resp) + } + return + } + resp.List[req.GroupID] = res + changeCount += len(res.Changes) + len(res.DeleteUserIds) + if changeCount > int(res.SyncCount)*4 { + break + } + } + apiresp.GinSuccess(c, resp) +} diff --git a/internal/api/router.go b/internal/api/router.go index 78e049e0f3..8f27336044 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -115,6 +115,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bfa7e3a574..bdea7b9ebd 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,9 +2,7 @@ package friend import ( "context" - "crypto/md5" - "encoding/binary" - "encoding/json" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbfriend "github.com/openimsdk/protocol/friend" @@ -59,49 +57,79 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr }, nil } -func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { - data, _ := json.Marshal(userIDs) - sum := md5.Sum(data) - return binary.BigEndian.Uint64(sum[:]) -} - func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - var limit int - if req.Version > 0 { - limit = s.config.RpcConfig.FriendSyncCount - } - incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) - if err != nil { - return nil, err + opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) + }, + Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) + if err != nil { + return nil, err + } + return friendsDB2PB(friends), nil + }, + ID: func(elem *pbfriend.FriendInfo) string { + return elem.FriendUserID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { + return &pbfriend.GetIncrementalFriendsResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + DeleteUserIds: delIDs, + Changes: list, + } + }, } - var ( - deleteUserIDs []string - changeUserIDs []string - ) - if incrVer.Full() { - changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - if err != nil { - return nil, err - } - } else { - deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() - } - var friends []*model.Friend - if len(changeUserIDs) > 0 { - friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) - if err != nil { - return nil, err - } - } - return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(incrVer.Version), - VersionID: incrVer.ID.Hex(), - Full: incrVer.Full(), - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - DeleteUserIds: deleteUserIDs, - Changes: friendsDB2PB(friends), - }, nil + return opt.Build() } + +//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +// if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { +// return nil, err +// } +// var limit int +// if req.Version > 0 { +// limit = s.config.RpcConfig.FriendSyncCount +// } +// incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) +// if err != nil { +// return nil, err +// } +// var ( +// deleteUserIDs []string +// changeUserIDs []string +// ) +// if incrVer.Full() { +// changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) +// if err != nil { +// return nil, err +// } +// } else { +// deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() +// } +// var friends []*model.Friend +// if len(changeUserIDs) > 0 { +// friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) +// if err != nil { +// return nil, err +// } +// } +// return &pbfriend.GetIncrementalFriendsResp{ +// Version: uint64(incrVer.Version), +// VersionID: incrVer.ID.Hex(), +// Full: incrVer.Full(), +// SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), +// DeleteUserIds: deleteUserIDs, +// Changes: friendsDB2PB(friends), +// }, nil +//} diff --git a/internal/rpc/group/convert.go b/internal/rpc/group/convert.go index a75693904d..8026430c30 100644 --- a/internal/rpc/group/convert.go +++ b/internal/rpc/group/convert.go @@ -57,3 +57,7 @@ func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel InviterUserID: member.InviterUserID, } } + +func (s *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { + return s.groupMemberDB2PB(member, 0) +} diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 51fd2d7b66..9b9ef07c13 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -77,6 +77,9 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { + if config.RpcConfig.GroupSyncCount <= 0 { + config.RpcConfig.GroupSyncCount = constant.MaxSyncPullNumber + } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -642,18 +645,29 @@ func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, req.UserIDs) + members, err := s.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupMembersInfoResp{ + Members: members, + }, nil +} + +func (s *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + members, err := s.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } if err := s.PopulateGroupMember(ctx, members...); err != nil { return nil, err } - return &pbgroup.GetGroupMembersInfoResp{ - Members: datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { - return convert.Db2PbGroupMember(e) - }), - }, nil + return datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { + return convert.Db2PbGroupMember(e) + }), nil } // GetGroupApplicationList handles functions that get a list of group requests. @@ -722,15 +736,28 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupID is empty") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := s.getGroupsInfo(ctx, req.GroupIDs) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupsInfoResp{ + GroupInfos: groups, + }, nil +} + +func (s *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { + if len(groupIDs) == 0 { + return nil, nil + } + groups, err := s.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, req.GroupIDs) + groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, req.GroupIDs) + owners, err := s.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } @@ -740,15 +767,13 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - return &pbgroup.GetGroupsInfoResp{ - GroupInfos: datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { - var ownerUserID string - if owner, ok := ownerMap[e.GroupID]; ok { - ownerUserID = owner.UserID - } - return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) - }), - }, nil + return datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { + var ownerUserID string + if owner, ok := ownerMap[e.GroupID]; ok { + ownerUserID = owner.UserID + } + return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) + }), nil } func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 765bceb632..80cb5a9eae 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,7 +2,11 @@ package group import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" ) func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { @@ -10,7 +14,63 @@ func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.Search panic("implement me") } -func (s *groupServer) GetGroupMemberHash(ctx context.Context, req *pbgroup.GetGroupMemberHashReq) (*pbgroup.GetGroupMemberHashResp, error) { - //TODO implement me - panic("implement me") +func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.db.FindMemberIncrVersion(ctx, req.GroupID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.db.FindSortGroupMemberUserIDs(ctx, req.GroupID) + }, + Find: func(ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + return s.getGroupMembersInfo(ctx, req.GroupID, ids) + }, + ID: func(elem *sdkws.GroupMemberFullInfo) string { + return elem.UserID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + return &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), + DeleteUserIds: delIDs, + Changes: list, + } + }, + } + return opt.Build() +} + +func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.db.FindJoinIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.db.FindSortJoinGroupIDs(ctx, req.UserID) + }, + Find: func(ids []string) ([]*sdkws.GroupInfo, error) { + return s.getGroupsInfo(ctx, ids) + }, + ID: func(elem *sdkws.GroupInfo) string { + return elem.GroupID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + return &pbgroup.GetIncrementalJoinGroupResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), + DeleteGroupIds: delIDs, + Changes: list, + } + }, + } + return opt.Build() } diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go new file mode 100644 index 0000000000..3e19056bfc --- /dev/null +++ b/internal/rpc/incrversion/option.go @@ -0,0 +1,58 @@ +package incrversion + +import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" +) + +func Limit(maxSync int, version uint64) int { + if version == 0 { + return 0 + } + return maxSync +} + +type Option[A, B any] struct { + VersionID string + Version func() (*model.VersionLog, error) + AllID func() ([]string, error) + Find func(ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +} + +func (o *Option[A, B]) Build() (*B, error) { + version, err := o.Version() + if err != nil { + return nil, err + } + var ( + deleteIDs []string + changeIDs []string + ) + full := o.VersionID != version.ID.Hex() || version.Full() + if full { + changeIDs, err = o.AllID() + if err != nil { + return nil, err + } + } else { + deleteIDs, changeIDs = version.DeleteAndChangeIDs() + } + var list []A + if len(changeIDs) > 0 { + list, err = o.Find(changeIDs) + if err != nil { + return nil, err + } + if (!full) && o.ID != nil && len(changeIDs) != len(list) { + foundIDs := datautil.SliceSetAny(list, o.ID) + for _, id := range changeIDs { + if _, ok := foundIDs[id]; !ok { + deleteIDs = append(deleteIDs, id) + } + } + } + } + return o.Resp(version, deleteIDs, list, full), nil +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da62acebf1..31a54c371f 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -254,7 +254,8 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + GroupSyncCount int `mapstructure:"groupSyncCount"` } type Msg struct { diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 53e2cd1c74..ff840a4a56 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -59,4 +59,7 @@ type GroupCache interface { GetGroupRolesLevelMemberInfo(ctx context.Context, groupID string, roleLevels []int32) ([]*model.GroupMember, error) GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache + + FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 2de03906f1..7f0ba62e44 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -46,6 +46,7 @@ type GroupCacheRedis struct { expireTime time.Duration rcClient *rockscache.Client groupHash cache.GroupHash + syncCount int } func NewGroupCacheRedis( @@ -406,3 +407,25 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st return g.groupMemberDB.Take(ctx, groupID, userID) }) } + +func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { + userIDs, err := g.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return nil, err + } + if len(userIDs) > g.syncCount { + userIDs = userIDs[:g.syncCount] + } + return userIDs, nil +} + +func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return nil, err + } + if len(groupIDs) > g.syncCount { + groupIDs = groupIDs[:g.syncCount] + } + return groupIDs, nil +} diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f2a1358357..c32f009ddc 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -106,6 +106,13 @@ type GroupDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) // DeleteGroupMemberHash deletes the hash entries for group members in specified groups. DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error + + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + + FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + + FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -219,10 +226,21 @@ func (g *groupDatabase) SearchGroup(ctx context.Context, keyword string, paginat } func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data map[string]any) error { - if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { - return err - } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { + return err + } + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { + return err + } + } + return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + }) } func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error { @@ -231,11 +249,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete if err := g.groupDB.UpdateStatus(ctx, groupID, constant.GroupStatusDismissed); err != nil { return err } + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } if deleteMember { - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return err - } if err := g.groupMemberDB.Delete(ctx, groupID, nil); err != nil { return err } @@ -246,6 +264,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupAllRoleLevel(groupID). DelGroupMembersInfo(groupID, userIDs...) } + if len(userIDs) > 0 { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { + return err + } + } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) } @@ -443,3 +466,19 @@ func (g *groupDatabase) DeleteGroupMemberHash(ctx context.Context, groupIDs []st } return c.ChainExecDel(ctx) } + +func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit) +} + +func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) +} + +func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { + return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) +} + +func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { + return g.cache.FindSortJoinGroupIDs(ctx, userID) +} diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index f57f2c3173..c397eda58a 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -34,4 +34,7 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 2df85a4302..83daf9ecd3 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -45,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := NewVersionLog(db.Collection("friend_owner_version_log")) + owner, err := NewVersionLog(db.Collection("friend_version")) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) } for ownerUserID, friendUserIDs := range mp { - if err := f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false); err != nil { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false); err != nil { return err } } @@ -79,7 +79,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, true) }) } @@ -95,7 +95,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, false) }) } @@ -185,7 +185,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false) }) } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ccca386e57..372babc9f9 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -40,15 +40,53 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - return &GroupMemberMgo{coll: coll}, nil + member, err := NewVersionLog(db.Collection("group_member_version")) + if err != nil { + return nil, err + } + join, err := NewVersionLog(db.Collection("group_join_version")) + if err != nil { + return nil, err + } + return &GroupMemberMgo{coll: coll, member: member, join: join}, nil } type GroupMemberMgo struct { - coll *mongo.Collection + coll *mongo.Collection + member database.VersionLog + join database.VersionLog +} + +func (g *GroupMemberMgo) sortBson() any { + return bson.D{{"role_level", -1}, {"create_time", -1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { - return mongoutil.InsertMany(ctx, g.coll, groupMembers) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, g.coll, groupMembers) + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.GroupID] = append(gms[member.GroupID], member.UserID) + } + for groupID, userIDs := range gms { + if err := g.member.IncrVersion(ctx, groupID, userIDs, false); err != nil { + return err + } + } + return nil + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.UserID] = append(gms[member.UserID], member.GroupID) + } + for userID, groupIDs := range gms { + if err := g.join.IncrVersion(ctx, userID, groupIDs, false); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) { @@ -56,24 +94,41 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s if len(userIDs) > 0 { filter["user_id"] = bson.M{"$in": userIDs} } - return mongoutil.DeleteMany(ctx, g.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteMany(ctx, g.coll, filter) + }, func() error { + return g.member.IncrVersion(ctx, groupID, userIDs, true) + }, func() error { + if len(userIDs) == 0 { + return nil + } + return g.member.IncrVersion(ctx, groupID, userIDs, true) + }) } func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error { - return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + return mongoutil.IncrVersion(func() error { + return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, true) + }, func() error { + return g.join.IncrVersion(ctx, groupID, []string{userID}, true) + }) } func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) { - return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) -} - -func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*model.GroupMember, err error) { - // TODO implement me - panic("implement me") + if len(data) == 0 { + return nil + } + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, false) + }) } func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.sortBson())) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -90,11 +145,11 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sortBson())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.sortBson())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { @@ -118,3 +173,15 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { _, ok := data["role_level"] return ok } + +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error { + return g.join.IncrVersion(ctx, userID, groupIDs, deleted) +} + +func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.member.FindChangeLog(ctx, groupID, version, limit) +} + +func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.join.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index fe123073e9..5629c5c00f 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -36,7 +36,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { return err } -func (l *VersionLogMgo) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index 6783cf4ff6..3450d2776e 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -12,7 +12,7 @@ const ( ) type VersionLog interface { - WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } From e9f46271ba13c5b379f743cad49f1c92c0bb37ec Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 14:30:26 +0800 Subject: [PATCH 030/147] sync --- internal/api/group.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/api/group.go b/internal/api/group.go index 7baa163233..62e508d681 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -20,6 +20,7 @@ import ( "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/log" ) type GroupApi rpcclient.Group @@ -145,7 +146,7 @@ func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { } func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { - type BatchIncrementalReq[A any] struct { + type BatchIncrementalReq struct { UserID string `json:"user_id"` List []*group.GetIncrementalGroupMemberReq `json:"list"` } @@ -172,6 +173,7 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { if len(resp.List) == 0 { apiresp.GinError(c, err) } else { + log.ZError(c, "group incr sync versopn", err, "groupID", req.GroupID, "success", len(resp.List)) apiresp.GinSuccess(c, resp) } return From 6363358ad41a52e11aa0e9e86d3e7e33d4bd84f5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 15:32:20 +0800 Subject: [PATCH 031/147] sync --- cmd/openim-rpc/openim-rpc-friend/main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-friend/main.go b/cmd/openim-rpc/openim-rpc-friend/main.go index 4589fb30d5..745c405532 100644 --- a/cmd/openim-rpc/openim-rpc-friend/main.go +++ b/cmd/openim-rpc/openim-rpc-friend/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewFriendRpcCmd().Exec(); err != nil { program.ExitWithError(err) } From 1b5621bc0933369c028885f1861141439d384298 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 18:44:42 +0800 Subject: [PATCH 032/147] group sync --- internal/rpc/group/group.go | 2 +- pkg/common/storage/cache/redis/group.go | 2 ++ pkg/common/storage/controller/group.go | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9b9ef07c13..94262da2c3 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -104,7 +104,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) var gs groupServer - database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) + database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs), config.RpcConfig.GroupSyncCount) gs.db = database gs.user = userRpcClient gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 7f0ba62e44..1dcace5ed1 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -57,6 +57,7 @@ func NewGroupCacheRedis( groupRequestDB database.GroupRequest, hashCode cache.GroupHash, opts *rockscache.Options, + syncCount int, ) cache.GroupCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Group.Topic}) g := localCache.Group @@ -70,6 +71,7 @@ func NewGroupCacheRedis( groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, groupHash: hashCode, + syncCount: syncCount, } } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index c32f009ddc..640ded6bd3 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -123,13 +123,14 @@ func NewGroupDatabase( groupRequestDB database.GroupRequest, ctxTx tx.Tx, groupHash cache.GroupHash, + syncCount int, ) GroupDatabase { return &groupDatabase{ groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, ctxTx: ctxTx, - cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()), + cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions(), syncCount), } } From 693935237a87a5a46d7aa8e76aac4a370c918569 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 4 Jun 2024 17:31:49 +0800 Subject: [PATCH 033/147] sync option --- internal/rpc/friend/sync.go | 62 ++------- internal/rpc/group/sync.go | 44 +++---- internal/rpc/incrversion/option.go | 131 +++++++++++++++++--- pkg/common/storage/cache/cachekey/friend.go | 23 ++-- pkg/common/storage/cache/cachekey/group.go | 10 ++ pkg/common/storage/cache/friend.go | 8 +- pkg/common/storage/cache/group.go | 5 + pkg/common/storage/cache/redis/friend.go | 53 +++++--- pkg/common/storage/cache/redis/group.go | 40 ++++++ pkg/common/storage/controller/friend.go | 37 ++++-- pkg/common/storage/controller/group.go | 79 ++++++++---- pkg/common/storage/model/version_log.go | 10 +- 12 files changed, 336 insertions(+), 166 deletions(-) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bdea7b9ebd..a99b0f220f 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -62,23 +62,22 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. return nil, err } opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - }, - Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.FriendSyncCount, + Version: s.friendDatabase.FindFriendIncrVersion, + CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, + SortID: s.friendDatabase.FindSortFriendUserIDs, + Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err } return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { - return elem.FriendUserID - }, + ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { return &pbfriend.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), @@ -92,44 +91,3 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } return opt.Build() } - -//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { -// if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { -// return nil, err -// } -// var limit int -// if req.Version > 0 { -// limit = s.config.RpcConfig.FriendSyncCount -// } -// incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) -// if err != nil { -// return nil, err -// } -// var ( -// deleteUserIDs []string -// changeUserIDs []string -// ) -// if incrVer.Full() { -// changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) -// if err != nil { -// return nil, err -// } -// } else { -// deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() -// } -// var friends []*model.Friend -// if len(changeUserIDs) > 0 { -// friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) -// if err != nil { -// return nil, err -// } -// } -// return &pbfriend.GetIncrementalFriendsResp{ -// Version: uint64(incrVer.Version), -// VersionID: incrVer.ID.Hex(), -// Full: incrVer.Full(), -// SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), -// DeleteUserIds: deleteUserIDs, -// Changes: friendsDB2PB(friends), -// }, nil -//} diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 80cb5a9eae..445ba68742 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -16,19 +16,18 @@ func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.Search func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.db.FindMemberIncrVersion(ctx, req.GroupID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.db.FindSortGroupMemberUserIDs(ctx, req.GroupID) - }, - Find: func(ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.GroupSyncCount, + Version: s.db.FindMemberIncrVersion, + CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, + SortID: s.db.FindSortGroupMemberUserIDs, + Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, - ID: func(elem *sdkws.GroupMemberFullInfo) string { - return elem.UserID - }, + ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ VersionID: version.ID.Hex(), @@ -48,19 +47,16 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. return nil, err } opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.db.FindJoinIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.db.FindSortJoinGroupIDs(ctx, req.UserID) - }, - Find: func(ids []string) ([]*sdkws.GroupInfo, error) { - return s.getGroupsInfo(ctx, ids) - }, - ID: func(elem *sdkws.GroupInfo) string { - return elem.GroupID - }, + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.GroupSyncCount, + Version: s.db.FindJoinIncrVersion, + CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, + SortID: s.db.FindSortJoinGroupIDs, + Find: s.getGroupsInfo, + ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index 3e19056bfc..3146985aa9 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -1,38 +1,139 @@ package incrversion import ( + "context" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson/primitive" ) -func Limit(maxSync int, version uint64) int { - if version == 0 { - return 0 +//func Limit(maxSync int, version uint64) int { +// if version == 0 { +// return 0 +// } +// return maxSync +//} + +const ( + tagQuery = iota + 1 + tagFull + tageEqual +) + +type Option[A, B any] struct { + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + SyncLimit int + CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) + Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +} + +func (o *Option[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *Option[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") } - return maxSync + if o.VersionKey == "" { + return o.newError("versionKey is empty") + } + if o.SyncLimit <= 0 { + return o.newError("invalid synchronization quantity") + } + if o.Version == nil { + return o.newError("func version is nil") + } + if o.SortID == nil { + return o.newError("func allID is nil") + } + if o.Find == nil { + return o.newError("func find is nil") + } + if o.ID == nil { + return o.newError("func id is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil } -type Option[A, B any] struct { - VersionID string - Version func() (*model.VersionLog, error) - AllID func() ([]string, error) - Find func(ids []string) ([]A, error) - ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +func (o *Option[A, B]) validVersion() bool { + objID, err := primitive.ObjectIDFromHex(o.VersionID) + return err == nil && (!objID.IsZero()) && o.VersionNumber > 0 +} + +func (o *Option[A, B]) equalID(objID primitive.ObjectID) bool { + return o.VersionID == objID.Hex() +} + +func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { + if o.CacheMaxVersion == nil { + if o.validVersion() { + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + } + *tag = tagFull + return o.Version(o.Ctx, o.VersionKey, 0, 0) + } else { + cache, err := o.CacheMaxVersion(o.Ctx, o.VersionKey) + if err != nil { + return nil, err + } + if !o.validVersion() { + *tag = tagFull + return cache, nil + } + if !o.equalID(cache.ID) { + *tag = tagFull + return cache, nil + } + if o.VersionNumber == uint64(cache.Version) { + *tag = tageEqual + return cache, nil + } + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + } } func (o *Option[A, B]) Build() (*B, error) { - version, err := o.Version() + if err := o.check(); err != nil { + return nil, err + } + var tag int + version, err := o.getVersion(&tag) if err != nil { return nil, err } + var full bool + switch tag { + case tagQuery: + full = version.ID.Hex() != o.VersionID || uint64(version.Version) < o.VersionNumber || len(version.Logs) != version.LogLen + case tagFull: + full = true + case tageEqual: + full = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } var ( deleteIDs []string changeIDs []string ) - full := o.VersionID != version.ID.Hex() || version.Full() + //full := o.VersionID != version.ID.Hex() || version.Full() if full { - changeIDs, err = o.AllID() + changeIDs, err = o.SortID(o.Ctx, o.VersionKey) if err != nil { return nil, err } @@ -41,7 +142,7 @@ func (o *Option[A, B]) Build() (*B, error) { } var list []A if len(changeIDs) > 0 { - list, err = o.Find(changeIDs) + list, err = o.Find(o.Ctx, changeIDs) if err != nil { return nil, err } diff --git a/pkg/common/storage/cache/cachekey/friend.go b/pkg/common/storage/cache/cachekey/friend.go index 6a217bdef9..8a053ca324 100644 --- a/pkg/common/storage/cache/cachekey/friend.go +++ b/pkg/common/storage/cache/cachekey/friend.go @@ -14,14 +14,13 @@ package cachekey -import "strconv" - const ( - FriendIDsKey = "FRIEND_IDS:" - TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" - FriendKey = "FRIEND_INFO:" - IsFriendKey = "IS_FRIEND:" // local cache key - FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendIDsKey = "FRIEND_IDS:" + TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" + FriendKey = "FRIEND_INFO:" + IsFriendKey = "IS_FRIEND:" // local cache key + //FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendMaxVersionKey = "FRIEND_MAX_VERSION:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -36,10 +35,14 @@ func GetFriendKey(ownerUserID, friendUserID string) string { return FriendKey + ownerUserID + "-" + friendUserID } +func GetFriendMaxVersionKey(ownerUserID string) string { + return FriendMaxVersionKey + ownerUserID +} + func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } -func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { - return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID -} +//func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { +// return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +//} diff --git a/pkg/common/storage/cache/cachekey/group.go b/pkg/common/storage/cache/cachekey/group.go index 681121ecba..2ef42c0ff4 100644 --- a/pkg/common/storage/cache/cachekey/group.go +++ b/pkg/common/storage/cache/cachekey/group.go @@ -28,6 +28,8 @@ const ( JoinedGroupsKey = "JOIN_GROUPS_KEY:" GroupMemberNumKey = "GROUP_MEMBER_NUM_CACHE:" GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:" + GroupMemberMaxVersionKey = "GROUP_MEMBER_MAX_VERSION:" + GroupJoinMaxVersionKey = "GROUP_JOIN_MAX_VERSION:" ) func GetGroupInfoKey(groupID string) string { @@ -57,3 +59,11 @@ func GetGroupMemberNumKey(groupID string) string { func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string { return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel)) } + +func GetGroupMemberMaxVersionKey(groupID string) string { + return GroupMemberMaxVersionKey + groupID +} + +func GetJoinGroupMaxVersionKey(userID string) string { + return GroupJoinMaxVersionKey + userID +} diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 3fee297ac8..064e3baae7 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -35,9 +35,13 @@ type FriendCache interface { DelOwner(friendUserID string, ownerUserIDs []string) FriendCache - DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + DelMaxFriendVersion(ownerUserIDs ...string) FriendCache + + //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + + FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index ff840a4a56..f02379a7dc 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -62,4 +62,9 @@ type GroupCache interface { FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + DelMaxGroupMemberVersion(groupIDs ...string) GroupCache + DelMaxJoinGroupVersion(userIDs ...string) GroupCache + FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index f4edbca9a3..62fad91cd1 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -70,8 +70,12 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } -func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { - return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { +// return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +//} + +func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { + return cachekey.GetFriendMaxVersionKey(ownerUserID) } // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. @@ -103,15 +107,15 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { - newGroupCache := f.CloneFriendCache() - keys := make([]string, 0, len(ownerUserIDs)) - for _, userID := range ownerUserIDs { - keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) - } - newGroupCache.AddKeys(keys...) - return newGroupCache -} +//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { +// newGroupCache := f.CloneFriendCache() +// keys := make([]string, 0, len(ownerUserIDs)) +// for _, userID := range ownerUserIDs { +// keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) +// } +// newGroupCache.AddKeys(keys...) +// return newGroupCache +//} // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { @@ -179,12 +183,29 @@ func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) return newFriendCache } +func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendMaxVersionKey(ownerUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { - return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) - }) + userIDs, err := f.GetFriendIDs(ctx, ownerUserID) + if err != nil { + return nil, err + } + if len(userIDs) > f.syncCount { + userIDs = userIDs[:f.syncCount] + } + return userIDs, nil } -func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { - return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) +func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) + }) } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 1dcace5ed1..cf88a65149 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -114,6 +114,14 @@ func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLeve return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel) } +func (g *GroupCacheRedis) getGroupMemberMaxVersionKey(groupID string) string { + return cachekey.GetGroupMemberMaxVersionKey(groupID) +} + +func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { + return cachekey.GetJoinGroupMaxVersionKey(userID) +} + func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { key := g.getGroupInfoKey(group.GroupID) for i, _key := range keys { @@ -431,3 +439,35 @@ func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID strin } return groupIDs, nil } + +func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(groupIDs)) + for _, groupID := range groupIDs { + keys = append(keys, g.getGroupMemberMaxVersionKey(groupID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + keys = append(keys, g.getJoinGroupMaxVersionKey(userID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, groupID, 0, 0) + }) +} + +func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 1af967b9bb..8f72703c00 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -82,6 +82,8 @@ type FriendDatabase interface { FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error @@ -185,7 +187,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return err } newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...).DelSortFriendUserIDs(ownerUserID) + cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...) return cache.ChainExecDel(ctx) }) @@ -288,7 +290,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelMaxFriendVersion(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -297,7 +299,8 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).DelSortFriendUserIDs(ownerUserID).ChainExecDel(ctx) + userIds := append(friendUserIDs, ownerUserID) + return f.cache.DelFriendIDs(userIds...).DelMaxFriendVersion(userIds...).ChainExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -305,7 +308,7 @@ func (f *friendDatabase) UpdateRemark(ctx context.Context, ownerUserID, friendUs if err := f.friend.UpdateRemark(ctx, ownerUserID, friendUserID, remark); err != nil { return err } - return f.cache.DelFriend(ownerUserID, friendUserID).ChainExecDel(ctx) + return f.cache.DelFriend(ownerUserID, friendUserID).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } // PageOwnerFriends retrieves the list of friends for the ownerUserID. It does not return an error if the result is empty. @@ -351,10 +354,12 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if len(val) == 0 { return nil } - if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { - return err - } - return f.cache.DelFriends(ownerUserID, friendUserIDs).DelSortFriendUserIDs(ownerUserID).ChainExecDel(ctx) + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return err + } + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) + }) } func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { @@ -362,7 +367,11 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID } func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { - return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) + return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) +} + +func (f *friendDatabase) FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return f.cache.FindMaxFriendVersion(ctx, ownerUserID) } func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { @@ -370,10 +379,12 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri } func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { - if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { - return err - } - return f.cache.DelOwner(friendUserID, ownerUserIDs).ChainExecDel(ctx) + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { + return err + } + return f.cache.DelOwner(friendUserID, ownerUserIDs).DelMaxFriendVersion(ownerUserIDs...).ChainExecDel(ctx) + }) } func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 640ded6bd3..9750b8b65f 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -113,6 +113,9 @@ type GroupDatabase interface { FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) } func NewGroupDatabase( @@ -182,7 +185,8 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMembersHash(group.GroupID). DelGroupsMemberNum(group.GroupID). DelGroupMemberIDs(group.GroupID). - DelGroupAllRoleLevel(group.GroupID) + DelGroupAllRoleLevel(group.GroupID). + DelMaxGroupMemberVersion(group.GroupID) } } if len(groupMembers) > 0 { @@ -195,7 +199,9 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMemberIDs(groupMember.GroupID). DelJoinedGroupID(groupMember.UserID). DelGroupMembersInfo(groupMember.GroupID, groupMember.UserID). - DelGroupAllRoleLevel(groupMember.GroupID) + DelGroupAllRoleLevel(groupMember.GroupID). + DelMaxJoinGroupVersion(groupMember.UserID). + DelMaxGroupMemberVersion(groupMember.GroupID) } } return c.ChainExecDel(ctx) @@ -239,8 +245,9 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { return err } + } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxJoinGroupVersion(userIDs...).ChainExecDel(ctx) }) } @@ -263,8 +270,10 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupsMemberNum(groupID). DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). - DelGroupMembersInfo(groupID, userIDs...) + DelGroupMembersInfo(groupID, userIDs...). + DelMaxGroupMemberVersion(groupID) } + c = c.DelMaxJoinGroupVersion(userIDs...) if len(userIDs) > 0 { if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { return err @@ -340,7 +349,9 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, DelGroupMemberIDs(groupID). DelGroupsMemberNum(groupID). DelJoinedGroupID(member.UserID). - DelGroupRoleLevel(groupID, []int32{member.RoleLevel}) + DelGroupRoleLevel(groupID, []int32{member.RoleLevel}). + DelMaxJoinGroupVersion(userID). + DelMaxGroupMemberVersion(groupID) if err := c.ChainExecDel(ctx); err != nil { return err } @@ -350,17 +361,20 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, } func (g *groupDatabase) DeleteGroupMember(ctx context.Context, groupID string, userIDs []string) error { - if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { - return err - } - c := g.cache.CloneGroupCache() - return c.DelGroupMembersHash(groupID). - DelGroupMemberIDs(groupID). - DelGroupsMemberNum(groupID). - DelJoinedGroupID(userIDs...). - DelGroupMembersInfo(groupID, userIDs...). - DelGroupAllRoleLevel(groupID). - ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { + return err + } + c := g.cache.CloneGroupCache() + return c.DelGroupMembersHash(groupID). + DelGroupMemberIDs(groupID). + DelGroupsMemberNum(groupID). + DelJoinedGroupID(userIDs...). + DelGroupMembersInfo(groupID, userIDs...). + DelGroupAllRoleLevel(groupID). + DelMaxGroupMemberVersion(groupID). + ChainExecDel(ctx) + }) } func (g *groupDatabase) MapGroupMemberUserID(ctx context.Context, groupIDs []string) (map[string]*common.GroupSimpleUserID, error) { @@ -390,20 +404,25 @@ func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, c := g.cache.CloneGroupCache() return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). - DelGroupMembersHash(groupID).ChainExecDel(ctx) + DelGroupMembersHash(groupID). + DelJoinedGroupID(oldOwnerUserID, newOwnerUserID). + ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { - if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { - return err - } - c := g.cache.CloneGroupCache() - c = c.DelGroupMembersInfo(groupID, userID) - if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) - } - return c.ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { + return err + } + c := g.cache.CloneGroupCache() + c = c.DelGroupMembersInfo(groupID, userID) + if g.groupMemberDB.IsUpdateRoleLevel(data) { + c = c.DelGroupAllRoleLevel(groupID) + } + c = c.DelMaxGroupMemberVersion(groupID).DelMaxJoinGroupVersion(userID) + return c.ChainExecDel(ctx) + }) } func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.BatchUpdateGroupMember) error { @@ -483,3 +502,11 @@ func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { return g.cache.FindSortJoinGroupIDs(ctx, userID) } + +func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { + return g.cache.FindMaxGroupMemberVersion(ctx, groupID) +} + +func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { + return g.cache.FindMaxJoinGroupVersion(ctx, userID) +} diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index 044bd42da4..a09f493a86 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -30,7 +30,6 @@ func (v *VersionLogTable) VersionLog() *VersionLog { Deleted: v.Deleted, LastUpdate: v.LastUpdate, LogLen: 0, - queryDoc: true, } } @@ -42,15 +41,10 @@ type VersionLog struct { Deleted uint `bson:"deleted"` LastUpdate time.Time `bson:"last_update"` LogLen int `bson:"log_len"` - queryDoc bool `bson:"-"` } -func (w *VersionLog) Full() bool { - return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen -} - -func (w *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { - for _, l := range w.Logs { +func (v *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { + for _, l := range v.Logs { if l.Deleted { delIds = append(delIds, l.EID) } else { From 7e13faaa984436e2b47dd22b830e7097957a39b7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 4 Jun 2024 19:11:16 +0800 Subject: [PATCH 034/147] sync option --- pkg/common/storage/cache/redis/group.go | 10 +++++- pkg/common/storage/controller/group.go | 10 ++++++ pkg/common/storage/database/group.go | 4 +++ pkg/common/storage/database/mgo/friend.go | 32 ++++++++++-------- pkg/common/storage/database/mgo/group.go | 33 +++++++++++++++++++ .../storage/database/mgo/group_member.go | 10 +++--- 6 files changed, 80 insertions(+), 19 deletions(-) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index cf88a65149..67e2460585 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -257,9 +257,17 @@ func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { return cache } +func (g *GroupCacheRedis) findUserJoinedGroupID(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + if err != nil { + return nil, err + } + return g.groupDB.FindJoinSortGroupID(ctx, groupIDs) +} + func (g *GroupCacheRedis) GetJoinedGroupIDs(ctx context.Context, userID string) (joinedGroupIDs []string, err error) { return getCache(ctx, g.rcClient, g.getJoinedGroupsKey(userID), g.expireTime, func(ctx context.Context) ([]string, error) { - return g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + return g.findUserJoinedGroupID(ctx, userID) }) } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 9750b8b65f..f9ee65955d 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -116,6 +116,8 @@ type GroupDatabase interface { FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) + + SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } func NewGroupDatabase( @@ -510,3 +512,11 @@ func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, grou func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { return g.cache.FindMaxJoinGroupVersion(ctx, userID) } + +func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + groupIDs, err := g.cache.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return 0, nil, err + } + return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) +} diff --git a/pkg/common/storage/database/group.go b/pkg/common/storage/database/group.go index 712db09d2d..7ef22f6c9a 100644 --- a/pkg/common/storage/database/group.go +++ b/pkg/common/storage/database/group.go @@ -32,4 +32,8 @@ type Group interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get Group total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) + + SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 83daf9ecd3..e0e30642c8 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -52,6 +52,10 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { return &FriendMgo{coll: coll, owner: owner}, nil } +func (f *FriendMgo) friendSort() any { + return bson.D{{"is_pinned", -1}, {"friend_nickname", 1}, {"create_time", 1}} +} + // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { return mongoutil.IncrVersion(func() error { @@ -145,13 +149,13 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}) + opt := options.Find().SetSort(f.friendSort()) return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit)) return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -197,7 +201,7 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( filter := bson.M{ "friend_user_id": friendUserID, } - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { @@ -209,14 +213,16 @@ func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID strin } func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - //where := bson.M{ - // "owner_user_id": ownerUserID, - // "$or": []bson.M{ - // {"remark": bson.M{"$regex": keyword, "$options": "i"}}, - // {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, - // {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, - // }, - //} - //return f.aggregatePagination(ctx, where, pagination) - panic("todo") + filter := bson.M{ + "owner_user_id": ownerUserID, + } + if keyword != "" { + filter["$or"] = []bson.M{ + {"remark": bson.M{"$regex": keyword, "$options": "i"}}, + {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, + {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, + } + } + opt := options.Find().SetSort(f.friendSort()) + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) } diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 48d24560bc..630bc02919 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -47,6 +47,10 @@ type GroupMgo struct { coll *mongo.Collection } +func (g *GroupMgo) sortGroup() any { + return bson.D{{"group_name", 1}, {"create_time", 1}} +} + func (g *GroupMgo) Create(ctx context.Context, groups []*model.Group) (err error) { return mongoutil.InsertMany(ctx, g.coll, groups) } @@ -126,3 +130,32 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (g *GroupMgo) FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) { + if len(groupIDs) < 2 { + return groupIDs, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + opt := options.Find().SetSort(g.sortGroup()).SetProjection(bson.M{"_id": 0, "group_id": 1}) + return mongoutil.Find[string](ctx, g.coll, filter, opt) +} + +func (g *GroupMgo) SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + if len(groupIDs) == 0 { + return 0, nil, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + if keyword != "" { + filter["group_name"] = bson.M{"$regex": keyword} + } + // Define the sorting options + opts := options.Find().SetSort(g.sortGroup()) + // Perform the search with pagination and sorting + return mongoutil.FindPage[*model.Group](ctx, g.coll, filter, pagination, opts) +} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 372babc9f9..cb64c87a41 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -57,7 +57,7 @@ type GroupMemberMgo struct { join database.VersionLog } -func (g *GroupMemberMgo) sortBson() any { +func (g *GroupMemberMgo) memberSort() any { return bson.D{{"role_level", -1}, {"create_time", -1}} } @@ -128,7 +128,7 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri } func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.sortBson())) + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -143,13 +143,13 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) } -func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { +func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (int64, []*model.GroupMember, error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sortBson())) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.memberSort())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.sortBson())) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { From 1796c3a9e4f5a8eff96bc027ad0f27906d8ea4aa Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 5 Jun 2024 11:18:14 +0800 Subject: [PATCH 035/147] refactor: replace `friend` package with `realtion`. --- go.mod | 2 +- go.sum | 2 + internal/api/friend.go | 42 ++++++++------ internal/api/router.go | 6 +- internal/rpc/friend/black.go | 21 +++---- internal/rpc/friend/callback.go | 24 ++++---- internal/rpc/friend/convert.go | 8 +-- internal/rpc/friend/friend.go | 89 +++++++++++++++++------------ internal/rpc/friend/notification.go | 15 ++--- internal/rpc/friend/sync.go | 29 +++++----- internal/rpc/user/user.go | 29 +++++----- pkg/rpcclient/friend.go | 20 +++---- 12 files changed, 159 insertions(+), 128 deletions(-) diff --git a/go.mod b/go.mod index db418bf144..26c61aed79 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.3 + github.com/openimsdk/protocol v0.0.69-alpha.4 github.com/openimsdk/tools v0.0.49-alpha.24 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 4588df18a7..1b0feaa348 100644 --- a/go.sum +++ b/go.sum @@ -288,6 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.4 h1:QJkOFV5Hlu7CbkHG5smeVw+5fx5DVkpNJWqlAOJxuIY= +github.com/openimsdk/protocol v0.0.69-alpha.4/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/friend.go b/internal/api/friend.go index 3af162a536..6912fdbbbc 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -17,7 +17,7 @@ package api import ( "github.com/gin-gonic/gin" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" ) @@ -28,73 +28,77 @@ func NewFriendApi(client rpcclient.Friend) FriendApi { } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.ApplyToAddFriend, o.Client, c) + a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.Call(friend.FriendClient.RespondFriendApply, o.Client, c) + a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.DeleteFriend, o.Client, c) + a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriendsApply, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.Call(friend.FriendClient.SetFriendRemark, o.Client, c) + a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.AddBlack, o.Client, c) + a2r.Call(relation.FriendClient.AddBlack, o.Client, c) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationBlacks, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.RemoveBlack, o.Client, c) + a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.ImportFriends, o.Client, c) + a2r.Call(relation.FriendClient.ImportFriends, o.Client, c) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.IsFriend, o.Client, c) + a2r.Call(relation.FriendClient.IsFriend, o.Client, c) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.Call(friend.FriendClient.GetFriendIDs, o.Client, c) + a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) + a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) + a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c) } func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) +} + +func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 8f27336044..445a24694c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,9 @@ package api import ( "fmt" + "net/http" + "strings" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" @@ -14,8 +17,6 @@ import ( "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "net/http" - "strings" ) func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { @@ -81,6 +82,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/add_black", f.AddBlack) friendRouterGroup.POST("/get_black_list", f.GetPaginationBlacks) friendRouterGroup.POST("/remove_black", f.RemoveBlack) + friendRouterGroup.POST("/get_incremental_blacks", f.GetIncrementalBlacks) friendRouterGroup.POST("/import_friend", f.ImportFriends) friendRouterGroup.POST("/is_friend", f.IsFriend) friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index caec08b7ac..218d1e7f85 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -16,16 +16,17 @@ package friend import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) -func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.GetPaginationBlacksReq) (resp *pbfriend.GetPaginationBlacksResp, err error) { +func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.GetPaginationBlacksReq) (resp *relation.GetPaginationBlacksResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -33,7 +34,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge if err != nil { return nil, err } - resp = &pbfriend.GetPaginationBlacksResp{} + resp = &relation.GetPaginationBlacksResp{} resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -42,18 +43,18 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge return resp, nil } -func (s *friendServer) IsBlack(ctx context.Context, req *pbfriend.IsBlackReq) (*pbfriend.IsBlackResp, error) { +func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (*relation.IsBlackResp, error) { in1, in2, err := s.blackDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } - resp := &pbfriend.IsBlackResp{} + resp := &relation.IsBlackResp{} resp.InUser1Blacks = in1 resp.InUser2Blacks = in2 return resp, nil } -func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlackReq) (*pbfriend.RemoveBlackResp, error) { +func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlackReq) (*relation.RemoveBlackResp, error) { if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -64,10 +65,10 @@ func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlac s.notificationSender.BlackDeletedNotification(ctx, req) - return &pbfriend.RemoveBlackResp{}, nil + return &relation.RemoveBlackResp{}, nil } -func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) (*pbfriend.AddBlackResp, error) { +func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) (*relation.AddBlackResp, error) { if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -87,5 +88,5 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) return nil, err } s.notificationSender.BlackAddedNotification(ctx, req) - return &pbfriend.AddBlackResp{}, nil + return &relation.AddBlackResp{}, nil } diff --git a/internal/rpc/friend/callback.go b/internal/rpc/friend/callback.go index 0610cdb78a..746ad21fa1 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/friend/callback.go @@ -16,14 +16,15 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" ) -func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { +func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *relation.DeleteFriendReq) { cbReq := &cbapi.CallbackAfterDeleteFriendReq{ CallbackCommand: cbapi.CallbackAfterDeleteFriendCommand, OwnerUserID: req.OwnerUserID, @@ -32,7 +33,7 @@ func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *conf s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterDeleteFriendResp{}, after) } -func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ApplyToAddFriendReq) error { +func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *relation.ApplyToAddFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendCommand, @@ -50,7 +51,7 @@ func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *confi }) } -func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.ApplyToAddFriendReq) { +func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *relation.ApplyToAddFriendReq) { cbReq := &cbapi.CallbackAfterAddFriendReq{ CallbackCommand: cbapi.CallbackAfterAddFriendCommand, FromUserID: req.FromUserID, @@ -61,8 +62,7 @@ func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config. s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *pbfriend.SetFriendRemarkReq) { - +func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *relation.SetFriendRemarkReq) { cbReq := &cbapi.CallbackAfterSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackAfterSetFriendRemarkCommand, OwnerUserID: req.OwnerUserID, @@ -73,7 +73,7 @@ func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *c s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *pbfriend.ImportFriendReq) { +func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *relation.ImportFriendReq) { cbReq := &cbapi.CallbackAfterImportFriendsReq{ CallbackCommand: cbapi.CallbackAfterImportFriendsCommand, OwnerUserID: req.OwnerUserID, @@ -83,7 +83,7 @@ func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *con s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *pbfriend.RemoveBlackReq) { +func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *relation.RemoveBlackReq) { cbReq := &cbapi.CallbackAfterRemoveBlackReq{ CallbackCommand: cbapi.CallbackAfterRemoveBlackCommand, OwnerUserID: req.OwnerUserID, @@ -93,7 +93,7 @@ func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *confi s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *pbfriend.SetFriendRemarkReq) error { +func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *relation.SetFriendRemarkReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackBeforeSetFriendRemarkCommand, @@ -112,7 +112,7 @@ func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before }) } -func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *pbfriend.AddBlackReq) error { +func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *relation.AddBlackReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddBlackReq{ CallbackCommand: cbapi.CallbackBeforeAddBlackCommand, @@ -124,7 +124,7 @@ func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config }) } -func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *pbfriend.RespondFriendApplyReq) error { +func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *relation.RespondFriendApplyReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendAgreeReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendAgreeCommand, @@ -138,7 +138,7 @@ func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before * }) } -func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ImportFriendReq) error { +func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *relation.ImportFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeImportFriendsReq{ CallbackCommand: cbapi.CallbackBeforeImportFriendsCommand, diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go index 0730df5291..1129dab34f 100644 --- a/internal/rpc/friend/convert.go +++ b/internal/rpc/friend/convert.go @@ -2,12 +2,12 @@ package friend import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/utils/datautil" ) -func friendDB2PB(db *model.Friend) *friend.FriendInfo { - return &friend.FriendInfo{ +func friendDB2PB(db *model.Friend) *relation.FriendInfo { + return &relation.FriendInfo{ OwnerUserID: db.OwnerUserID, FriendUserID: db.FriendUserID, FriendNickname: db.FriendNickname, @@ -21,6 +21,6 @@ func friendDB2PB(db *model.Friend) *friend.FriendInfo { } } -func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { +func friendsDB2PB(db []*model.Friend) []*relation.FriendInfo { return datautil.Slice(db, friendDB2PB) } diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 1c6b52b141..a808812789 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -30,7 +31,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -54,7 +55,7 @@ type Config struct { RpcConfig config.Friend RedisConfig config.Redis MongodbConfig config.Mongo - //ZookeeperConfig config.ZooKeeper + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks @@ -103,7 +104,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) // Register Friend server with refactored MongoDB and Redis integrations - pbfriend.RegisterFriendServer(server, &friendServer{ + relation.RegisterFriendServer(server, &friendServer{ friendDatabase: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, @@ -126,8 +127,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // ok. -func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) (resp *pbfriend.ApplyToAddFriendResp, err error) { - resp = &pbfriend.ApplyToAddFriendResp{} +func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.ApplyToAddFriendReq) (resp *relation.ApplyToAddFriendResp, err error) { + resp = &relation.ApplyToAddFriendResp{} if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -157,7 +158,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply } // ok. -func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFriendReq) (resp *pbfriend.ImportFriendResp, err error) { +func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFriendReq) (resp *relation.ImportFriendResp, err error) { if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -179,7 +180,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr return nil, err } for _, userID := range req.FriendUserIDs { - s.notificationSender.FriendApplicationAgreedNotification(ctx, &pbfriend.RespondFriendApplyReq{ + s.notificationSender.FriendApplicationAgreedNotification(ctx, &relation.RespondFriendApplyReq{ FromUserID: req.OwnerUserID, ToUserID: userID, HandleResult: constant.FriendResponseAgree, @@ -187,12 +188,12 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr } s.webhookAfterImportFriends(ctx, &s.config.WebhooksConfig.AfterImportFriends, req) - return &pbfriend.ImportFriendResp{}, nil + return &relation.ImportFriendResp{}, nil } // ok. -func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.RespondFriendApplyReq) (resp *pbfriend.RespondFriendApplyResp, err error) { - resp = &pbfriend.RespondFriendApplyResp{} +func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.RespondFriendApplyReq) (resp *relation.RespondFriendApplyResp, err error) { + resp = &relation.RespondFriendApplyResp{} if err := authverify.CheckAccessV3(ctx, req.ToUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -226,8 +227,8 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res } // ok. -func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFriendReq) (resp *pbfriend.DeleteFriendResp, err error) { - resp = &pbfriend.DeleteFriendResp{} +func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFriendReq) (resp *relation.DeleteFriendResp, err error) { + resp = &relation.DeleteFriendResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -244,11 +245,11 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFri } // ok. -func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFriendRemarkReq) (resp *pbfriend.SetFriendRemarkResp, err error) { +func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFriendRemarkReq) (resp *relation.SetFriendRemarkResp, err error) { if err = s.webhookBeforeSetFriendRemark(ctx, &s.config.WebhooksConfig.BeforeSetFriendRemark, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - resp = &pbfriend.SetFriendRemarkResp{} + resp = &relation.SetFriendRemarkResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -265,8 +266,8 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFri } // ok. -func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.GetDesignatedFriendsReq) (resp *pbfriend.GetDesignatedFriendsResp, err error) { - resp = &pbfriend.GetDesignatedFriendsResp{} +func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { + resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } @@ -282,12 +283,13 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.G // Get the list of friend requests sent out proactively. func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, - req *pbfriend.GetDesignatedFriendsApplyReq) (resp *pbfriend.GetDesignatedFriendsApplyResp, err error) { + req *relation.GetDesignatedFriendsApplyReq, +) (resp *relation.GetDesignatedFriendsApplyResp, err error) { friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } - resp = &pbfriend.GetDesignatedFriendsApplyResp{} + resp = &relation.GetDesignatedFriendsApplyResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -296,7 +298,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, } // Get received friend requests (i.e., those initiated by others). -func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyToReq) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) { +func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *relation.GetPaginationFriendsApplyToReq) (resp *relation.GetPaginationFriendsApplyToResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -304,7 +306,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbf if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsApplyToResp{} + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -313,8 +315,8 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbf return resp, nil } -func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyFromReq) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) { - resp = &pbfriend.GetPaginationFriendsApplyFromResp{} +func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { + resp = &relation.GetPaginationFriendsApplyFromResp{} if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -331,8 +333,8 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *p } // ok. -func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) (resp *pbfriend.IsFriendResp, err error) { - resp = &pbfriend.IsFriendResp{} +func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { + resp = &relation.IsFriendResp{} resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err @@ -340,7 +342,7 @@ func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) return resp, nil } -func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.GetPaginationFriendsReq) (resp *pbfriend.GetPaginationFriendsResp, err error) { +func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.GetPaginationFriendsReq) (resp *relation.GetPaginationFriendsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -348,7 +350,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsResp{} + resp = &relation.GetPaginationFriendsResp{} resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -357,11 +359,11 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G return resp, nil } -func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriendIDsReq) (resp *pbfriend.GetFriendIDsResp, err error) { +func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriendIDsReq) (resp *relation.GetFriendIDsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - resp = &pbfriend.GetFriendIDsResp{} + resp = &relation.GetFriendIDsResp{} resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err @@ -369,7 +371,7 @@ func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriend return resp, nil } -func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfriend.GetSpecifiedFriendsInfoReq) (*pbfriend.GetSpecifiedFriendsInfoResp, error) { +func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relation.GetSpecifiedFriendsInfoReq) (*relation.GetSpecifiedFriendsInfoResp, error) { if len(req.UserIDList) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDList is empty") } @@ -394,8 +396,8 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien blackMap := datautil.SliceToMap(blacks, func(e *model.Black) string { return e.BlockUserID }) - resp := &pbfriend.GetSpecifiedFriendsInfoResp{ - Infos: make([]*pbfriend.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), + resp := &relation.GetSpecifiedFriendsInfoResp{ + Infos: make([]*relation.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), } for _, userID := range req.UserIDList { user := userMap[userID] @@ -404,7 +406,6 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } var friendInfo *sdkws.FriendInfo if friend := friendMap[userID]; friend != nil { - friendInfo = &sdkws.FriendInfo{ OwnerUserID: friend.OwnerUserID, Remark: friend.Remark, @@ -425,7 +426,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien Ex: black.Ex, } } - resp.Infos = append(resp.Infos, &pbfriend.GetSpecifiedFriendsInfoInfo{ + resp.Infos = append(resp.Infos, &relation.GetSpecifiedFriendsInfoInfo{ UserInfo: user, FriendInfo: friendInfo, BlackInfo: blackInfo, @@ -433,10 +434,11 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, - req *pbfriend.UpdateFriendsReq, -) (*pbfriend.UpdateFriendsResp, error) { + req *relation.UpdateFriendsReq, +) (*relation.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("friendIDList is empty") } @@ -464,8 +466,23 @@ func (s *friendServer) UpdateFriends( return nil, err } - resp := &pbfriend.UpdateFriendsResp{} + resp := &relation.UpdateFriendsResp{} s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } + +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *relation.GetIncrementalFriendsApplyToReq) (*relation.GetIncrementalFriendsApplyToResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *relation.GetIncrementalFriendsApplyFromReq) (*relation.GetIncrementalFriendsApplyFromResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *relation.GetIncrementalBlacksReq) (*relation.GetIncrementalBlacksResp, error) { + // TODO implement me + return nil, nil +} diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index 8089a9bdc8..ddee025bb6 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -16,6 +16,7 @@ package friend import ( "context" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -24,7 +25,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) @@ -127,7 +128,7 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) { +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, ToUserID: req.ToUserID, @@ -137,7 +138,7 @@ func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context. func (f *FriendNotificationSender) FriendApplicationAgreedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -148,7 +149,7 @@ func (f *FriendNotificationSender) FriendApplicationAgreedNotification( func (f *FriendNotificationSender) FriendApplicationRefusedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -182,7 +183,7 @@ func (f *FriendNotificationSender) FriendAddedNotification( return nil } -func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *pbfriend.DeleteFriendReq) { +func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *relation.DeleteFriendReq) { tips := sdkws.FriendDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.FriendUserID, @@ -204,14 +205,14 @@ func (f *FriendNotificationSender) FriendsInfoUpdateNotification(ctx context.Con f.Notification(ctx, toUserID, toUserID, constant.FriendsInfoUpdateNotification, &tips) } -func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *pbfriend.AddBlackReq) { +func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *relation.AddBlackReq) { tips := sdkws.BlackAddedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = req.OwnerUserID tips.FromToUserID.ToUserID = req.BlackUserID f.Notification(ctx, req.OwnerUserID, req.BlackUserID, constant.BlackAddedNotification, &tips) } -func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *pbfriend.RemoveBlackReq) { +func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *relation.RemoveBlackReq) { blackDeletedTips := sdkws.BlackDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.BlackUserID, diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bdea7b9ebd..ee8effa70d 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,14 +2,15 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/errs" ) -func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { if req.NewUserInfo == nil { var err error req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) @@ -30,10 +31,10 @@ func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfr } s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) } - return &pbfriend.NotificationUserInfoUpdateResp{}, nil + return &relation.NotificationUserInfoUpdateResp{}, nil } -func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { +func (s *friendServer) SearchFriends(ctx context.Context, req *relation.SearchFriendsReq) (*relation.SearchFriendsResp, error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -42,7 +43,7 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr if err != nil { return nil, err } - return &pbfriend.SearchFriendsResp{ + return &relation.SearchFriendsResp{ Total: total, Friends: friendsDB2PB(friends), }, nil @@ -51,17 +52,17 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr if err != nil { return nil, err } - return &pbfriend.SearchFriendsResp{ + return &relation.SearchFriendsResp{ Total: total, Friends: friendsDB2PB(friends), }, nil } -func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*relation.FriendInfo, relation.GetIncrementalFriendsResp]{ VersionID: req.VersionID, Version: func() (*model.VersionLog, error) { return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) @@ -69,18 +70,18 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. AllID: func() ([]string, error) { return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) }, - Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + Find: func(ids []string) ([]*relation.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err } return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { + ID: func(elem *relation.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + Resp: func(version *model.VersionLog, delIDs []string, list []*relation.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, @@ -93,7 +94,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. return opt.Build() } -//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { // if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { // return nil, err // } @@ -124,7 +125,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. // return nil, err // } // } -// return &pbfriend.GetIncrementalFriendsResp{ +// return &relation.GetIncrementalFriendsResp{ // Version: uint64(incrVer.Version), // VersionID: incrVer.ID.Hex(), // Full: incrVer.Full(), diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 504c08a2ca..35870d4d03 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,6 +17,11 @@ package user import ( "context" "errors" + "math/rand" + "strings" + "sync" + "time" + "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -24,13 +29,9 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - friendpb "github.com/openimsdk/protocol/friend" "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" - "math/rand" - "strings" - "sync" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -123,8 +124,7 @@ func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesig // deprecated: -//UpdateUserInfo - +// UpdateUserInfo func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -162,6 +162,7 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI } return resp, nil } + func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -198,6 +199,7 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse } return resp, nil } + func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { resp = &pbuser.SetGlobalRecvMessageOptResp{} if _, err := s.db.FindWithError(ctx, []string{req.UserID}); err != nil { @@ -256,7 +258,6 @@ func (s *userServer) GetPaginationUsers(ctx context.Context, req *pbuser.GetPagi return &pbuser.GetPaginationUsersResp{Total: int32(total), Users: convert.UsersDB2Pb(users)}, err } - } func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterReq) (resp *pbuser.UserRegisterResp, err error) { @@ -353,7 +354,8 @@ func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbus // GetUserStatus Get the online status of the user. func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { + err error, +) { onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) if err != nil { return nil, err @@ -363,7 +365,8 @@ func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatu // SetUserStatus Synchronize user's online status. func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { + err error, +) { err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) if err != nil { return nil, err @@ -387,7 +390,8 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + req *pbuser.GetSubscribeUsersStatusReq, +) (*pbuser.GetSubscribeUsersStatusResp, error) { userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) if err != nil { return nil, err @@ -476,7 +480,6 @@ func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.P } func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.ProcessUserCommandGetReq) (*pbuser.ProcessUserCommandGetResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err @@ -723,7 +726,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri go func() { defer wg.Done() - _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &relation.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go index 5543afe4f7..fd00be3292 100644 --- a/pkg/rpcclient/friend.go +++ b/pkg/rpcclient/friend.go @@ -17,7 +17,7 @@ package rpcclient import ( "context" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" sdkws "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/system/program" @@ -26,7 +26,7 @@ import ( type Friend struct { conn grpc.ClientConnInterface - Client friend.FriendClient + Client relation.FriendClient discov discovery.SvcDiscoveryRegistry } @@ -35,7 +35,7 @@ func NewFriend(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *F if err != nil { program.ExitWithError(err) } - client := friend.NewFriendClient(conn) + client := relation.NewFriendClient(conn) return &Friend{discov: discov, conn: conn, Client: client} } @@ -47,11 +47,11 @@ func NewFriendRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName s func (f *FriendRpcClient) GetFriendsInfo( ctx context.Context, - ownerUserID, friendUserID string, + ownerUserID, relationUserID string, ) (resp *sdkws.FriendInfo, err error) { r, err := f.Client.GetDesignatedFriends( ctx, - &friend.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{friendUserID}}, + &relation.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{relationUserID}}, ) if err != nil { return nil, err @@ -60,17 +60,17 @@ func (f *FriendRpcClient) GetFriendsInfo( return } -// possibleFriendUserID Is PossibleFriendUserId's friends. +// possibleFriendUserID Is PossibleFriendUserId's relations. func (f *FriendRpcClient) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (bool, error) { - resp, err := f.Client.IsFriend(ctx, &friend.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) + resp, err := f.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) if err != nil { return false, err } return resp.InUser1Friends, nil } -func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) { - req := friend.GetFriendIDsReq{UserID: ownerUserID} +func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (relationIDs []string, err error) { + req := relation.GetFriendIDsReq{UserID: ownerUserID} resp, err := f.Client.GetFriendIDs(ctx, &req) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) } func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := b.Client.IsBlack(ctx, &friend.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) + r, err := b.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) if err != nil { return false, err } From e11807c02874a670fb038a9e45e05d3d37c2c7c0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 5 Jun 2024 11:39:10 +0800 Subject: [PATCH 036/147] refactor: update lastest commit to relation. --- internal/rpc/friend/sync.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index ba689c7bac..6859826209 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -63,7 +63,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*relation.FriendInfo, relation.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -72,8 +72,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, SortID: s.friendDatabase.FindSortFriendUserIDs, - Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { - + Find: func(ctx context.Context, ids []string) ([]*relation.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err @@ -81,9 +80,9 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + ID: func(elem *relation.FriendInfo) string { return elem.FriendUserID }, + Resp: func(version *model.VersionLog, delIDs []string, list []*relation.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, From 6285b688a275460d2b9f814439c2bec8f75be63e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 09:45:44 +0800 Subject: [PATCH 037/147] sync option --- go.mod | 2 +- internal/api/group.go | 2 +- internal/rpc/friend/black.go | 2 +- internal/rpc/friend/callback.go | 2 +- internal/rpc/friend/convert.go | 26 ------ internal/rpc/friend/friend.go | 34 ++++++- internal/rpc/friend/notification.go | 2 +- internal/rpc/friend/sync.go | 88 +++++++----------- internal/rpc/group/sync.go | 5 -- internal/rpc/user/user.go | 105 +++++++--------------- pkg/common/storage/cache/group.go | 1 - pkg/common/storage/cache/redis/group.go | 21 ----- pkg/common/storage/controller/friend.go | 19 +--- pkg/common/storage/controller/user.go | 6 ++ pkg/common/storage/database/friend.go | 4 +- pkg/common/storage/database/mgo/friend.go | 40 ++++----- pkg/common/storage/database/mgo/user.go | 66 ++++++++++++++ pkg/common/storage/database/user.go | 3 + 18 files changed, 193 insertions(+), 235 deletions(-) delete mode 100644 internal/rpc/friend/convert.go diff --git a/go.mod b/go.mod index db418bf144..a3777939b3 100644 --- a/go.mod +++ b/go.mod @@ -178,4 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/internal/api/group.go b/internal/api/group.go index 62e508d681..5b8b9d0442 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -180,7 +180,7 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { } resp.List[req.GroupID] = res changeCount += len(res.Changes) + len(res.DeleteUserIds) - if changeCount > int(res.SyncCount)*4 { + if changeCount >= int(res.SyncCount) { break } } diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index caec08b7ac..18bb3dc0a6 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -21,7 +21,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) diff --git a/internal/rpc/friend/callback.go b/internal/rpc/friend/callback.go index 0610cdb78a..7677337345 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/friend/callback.go @@ -20,7 +20,7 @@ import ( cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" ) func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go deleted file mode 100644 index 0730df5291..0000000000 --- a/internal/rpc/friend/convert.go +++ /dev/null @@ -1,26 +0,0 @@ -package friend - -import ( - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/tools/utils/datautil" -) - -func friendDB2PB(db *model.Friend) *friend.FriendInfo { - return &friend.FriendInfo{ - OwnerUserID: db.OwnerUserID, - FriendUserID: db.FriendUserID, - FriendNickname: db.FriendNickname, - FriendFaceURL: db.FriendFaceURL, - Remark: db.Remark, - CreateTime: db.CreateTime.UnixMilli(), - AddSource: db.AddSource, - OperatorUserID: db.OperatorUserID, - Ex: db.Ex, - IsPinned: db.IsPinned, - } -} - -func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { - return datautil.Slice(db, friendDB2PB) -} diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 1c6b52b141..7eb10aa170 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -30,7 +30,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -50,6 +50,21 @@ type friendServer struct { webhookClient *webhook.Client } +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *pbfriend.GetIncrementalFriendsApplyToReq) (*pbfriend.GetIncrementalFriendsApplyToResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *pbfriend.GetIncrementalFriendsApplyFromReq) (*pbfriend.GetIncrementalFriendsApplyFromResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *pbfriend.GetIncrementalBlacksReq) (*pbfriend.GetIncrementalBlacksResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.Friend RedisConfig config.Redis @@ -270,14 +285,24 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.G if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + friends, err := s.getFriend(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } - if resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap); err != nil { + return &pbfriend.GetDesignatedFriendsResp{ + FriendsInfo: friends, + }, nil +} + +func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friendUserIDs []string) ([]*sdkws.FriendInfo, error) { + if len(friendUserIDs) == 0 { + return nil, nil + } + friends, err := s.friendDatabase.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + if err != nil { return nil, err } - return resp, nil + return convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. @@ -433,6 +458,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, req *pbfriend.UpdateFriendsReq, diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index 8089a9bdc8..9cf4345554 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -24,7 +24,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a99b0f220f..213a38151d 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -5,63 +5,39 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - pbfriend "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/tools/errs" + pbfriend "github.com/openimsdk/protocol/relation" + "github.com/openimsdk/protocol/sdkws" ) -func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { - if req.NewUserInfo == nil { - var err error - req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) - if err != nil { - return nil, err - } - } - if req.UserID != req.NewUserInfo.UserID { - return nil, errs.ErrArgs.WrapMsg("req.UserID != req.NewUserInfo.UserID") - } - userIDs, err := s.friendDatabase.FindFriendUserID(ctx, req.UserID) - if err != nil { - return nil, err - } - if len(userIDs) > 0 { - if err := s.friendDatabase.UpdateFriendUserInfo(ctx, req.UserID, userIDs, req.NewUserInfo.Nickname, req.NewUserInfo.FaceURL); err != nil { - return nil, err - } - s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) - } - return &pbfriend.NotificationUserInfoUpdateResp{}, nil -} - -func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { - if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { - return nil, err - } - if req.Keyword == "" { - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) - if err != nil { - return nil, err - } - return &pbfriend.SearchFriendsResp{ - Total: total, - Friends: friendsDB2PB(friends), - }, nil - } - total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) - if err != nil { - return nil, err - } - return &pbfriend.SearchFriendsResp{ - Total: total, - Friends: friendsDB2PB(friends), - }, nil -} +//func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { +// if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { +// return nil, err +// } +// if req.Keyword == "" { +// total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) +// if err != nil { +// return nil, err +// } +// return &pbfriend.SearchFriendsResp{ +// Total: total, +// Friends: friendsDB2PB(friends), +// }, nil +// } +// total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) +// if err != nil { +// return nil, err +// } +// return &pbfriend.SearchFriendsResp{ +// Total: total, +// Friends: friendsDB2PB(friends), +// }, nil +//} func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*sdkws.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -70,15 +46,11 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, SortID: s.friendDatabase.FindSortFriendUserIDs, - Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) - if err != nil { - return nil, err - } - return friendsDB2PB(friends), nil + Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { + return s.getFriend(ctx, req.UserID, ids) }, - ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { + ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { return &pbfriend.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 445ba68742..6e0103c83f 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -9,11 +9,6 @@ import ( "github.com/openimsdk/protocol/sdkws" ) -func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { - //TODO implement me - panic("implement me") -} - func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 504c08a2ca..db44be556f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,7 +16,6 @@ package user import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -24,12 +23,9 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - friendpb "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/db/redisutil" "math/rand" "strings" - "sync" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -135,29 +131,26 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } + data := convert.UserPb2DBMap(req.UserInfo) - oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) - if err != nil { + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + if err != nil { return nil, err } - //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - //if err != nil { - // return nil, err - //} - //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { - // return nil, err - // } - //} - //for _, friendID := range friends { - // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - //} + if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + return nil, err + } + } + for _, friendID := range friends { + s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + } s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { + if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { return nil, err } return resp, nil @@ -171,29 +164,25 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } - oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } - //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - //if err != nil { - // return nil, err - //} - //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - // return nil, err - // } - //} - //for _, friendID := range friends { - // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - //} + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } + if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + return nil, err + } + } + for _, friendID := range friends { + s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + } s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { + if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { return nil, err } return resp, nil @@ -695,40 +684,10 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } -func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { - user, err := s.db.GetUserByID(ctx, userID) +func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { + users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) if err != nil { - return err + return nil, err } - if *user == *oldUser { - return nil - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, userID) - if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { - return nil - } - oldUserInfo := convert.UserDB2Pb(oldUser) - newUserInfo := convert.UserDB2Pb(user) - var wg sync.WaitGroup - var es [2]error - wg.Add(len(es)) - go func() { - defer wg.Done() - _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ - UserID: userID, - OldUserInfo: oldUserInfo, - NewUserInfo: newUserInfo, - }) - }() - - go func() { - defer wg.Done() - _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ - UserID: userID, - OldUserInfo: oldUserInfo, - NewUserInfo: newUserInfo, - }) - }() - wg.Wait() - return errors.Join(es[:]...) + return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index f02379a7dc..00a46aaf82 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -46,7 +46,6 @@ type GroupCache interface { GetGroupMemberInfo(ctx context.Context, groupID, userID string) (groupMember *model.GroupMember, err error) GetGroupMembersInfo(ctx context.Context, groupID string, userID []string) (groupMembers []*model.GroupMember, err error) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) - GetGroupMembersPage(ctx context.Context, groupID string, userID []string, showNumber, pageNumber int32) (total uint32, groupMembers []*model.GroupMember, err error) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 67e2460585..9699230efc 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -27,7 +27,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" ) @@ -296,26 +295,6 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin }) } -func (g *GroupCacheRedis) GetGroupMembersPage( - ctx context.Context, - groupID string, - userIDs []string, - showNumber, pageNumber int32, -) (total uint32, groupMembers []*model.GroupMember, err error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return 0, nil, err - } - if userIDs != nil { - userIDs = datautil.BothExist(userIDs, groupMemberIDs) - } else { - userIDs = groupMemberIDs - } - groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber))) - - return uint32(len(userIDs)), groupMembers, err -} - func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) { groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) if err != nil { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 8f72703c00..e03208df9e 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -86,9 +86,7 @@ type FriendDatabase interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error - - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) } type friendDatabase struct { @@ -378,15 +376,6 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri return f.friend.FindFriendUserID(ctx, friendUserID) } -func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { - return f.tx.Transaction(ctx, func(ctx context.Context) error { - if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { - return err - } - return f.cache.DelOwner(friendUserID, ownerUserIDs).DelMaxFriendVersion(ownerUserIDs...).ChainExecDel(ctx) - }) -} - -func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) -} +//func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +//} diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 09dc2db22b..9efe535c01 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -60,6 +60,8 @@ type UserDatabase interface { CountTotal(ctx context.Context, before *time.Time) (int64, error) // CountRangeEverydayTotal Get the user increment in the range CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) // SubscribeUsersStatus Subscribe a user's presence status SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error // UnsubscribeUsersStatus unsubscribe a user's presence status @@ -210,6 +212,10 @@ func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.T return u.userDB.CountRangeEverydayTotal(ctx, start, end) } +func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + return u.userDB.SortQuery(ctx, userIDName, asc) +} + // SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 6ab1185bc0..1c9cd10330 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -52,9 +52,7 @@ type Friend interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error - - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index e0e30642c8..646951d7d0 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -53,7 +53,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"friend_nickname", 1}, {"create_time", 1}} + return bson.D{{"is_pinned", -1}, {"create_time", 1}} } // Create inserts multiple friend records. @@ -168,7 +168,7 @@ func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { @@ -204,25 +204,17 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } -func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { - filter := bson.M{ - "friend_user_id": friendUserID, - } - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, bson.M{"$set": bson.M{"nickname": nickname, "face_url": faceURL}}) - return err -} - -func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - filter := bson.M{ - "owner_user_id": ownerUserID, - } - if keyword != "" { - filter["$or"] = []bson.M{ - {"remark": bson.M{"$regex": keyword, "$options": "i"}}, - {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, - {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, - } - } - opt := options.Find().SetSort(f.friendSort()) - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) -} +//func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// filter := bson.M{ +// "owner_user_id": ownerUserID, +// } +// if keyword != "" { +// filter["$or"] = []bson.M{ +// {"remark": bson.M{"$regex": keyword, "$options": "i"}}, +// {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, +// {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, +// } +// } +// opt := options.Find().SetSort(f.friendSort()) +// return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) +//} diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 96cb18882b..793b8cdc85 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -319,3 +319,69 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (u *UserMgo) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + if len(userIDName) == 0 { + return nil, nil + } + userIDs := make([]string, 0, len(userIDName)) + attached := make(map[string]string) + for userID, name := range userIDName { + userIDs = append(userIDs, userID) + if name == "" { + continue + } + attached[userID] = name + } + var sortValue int + if asc { + sortValue = 1 + } else { + sortValue = -1 + } + if len(attached) == 0 { + filter := bson.M{"user_id": bson.M{"$in": userIDs}} + opt := options.Find().SetSort(bson.M{"nickname": sortValue}) + return mongoutil.Find[*model.User](ctx, u.coll, filter, opt) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": bson.M{"$in": userIDs}, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$arrayElemAt": []any{ + bson.M{ + "$filter": bson.M{ + "input": bson.M{ + "$objectToArray": attached, + }, + "as": "item", + "cond": bson.M{ + "$eq": []any{"$$item.k", "$user_id"}, + }, + }, + }, + 0, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$ifNull": []any{"$_query_sort_name.v", "$nickname"}, + }, + }, + }, + { + "$sort": bson.M{ + "_query_sort_name": sortValue, + }, + }, + } + return mongoutil.Aggregate[*model.User](ctx, u.coll, pipeline) +} diff --git a/pkg/common/storage/database/user.go b/pkg/common/storage/database/user.go index 2e40886205..4ddc8285f6 100644 --- a/pkg/common/storage/database/user.go +++ b/pkg/common/storage/database/user.go @@ -39,6 +39,9 @@ type User interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get user total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) + // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error From c5f565ff209740e35a81976e427c4fbb26d7d96e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 09:57:47 +0800 Subject: [PATCH 038/147] sync option --- internal/rpc/friend/convert.go | 0 internal/rpc/friend/friend.go | 2 +- internal/rpc/friend/sync.go | 8 ++++---- internal/rpc/user/user.go | 4 ---- 4 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 internal/rpc/friend/convert.go diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 51fae71c48..cbc16cca96 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -275,7 +275,7 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.G if err != nil { return nil, err } - return &pbfriend.GetDesignatedFriendsResp{ + return &relation.GetDesignatedFriendsResp{ FriendsInfo: friends, }, nil } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 46bc1d66a4..9af1c47d32 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,12 +2,12 @@ package friend import ( "context" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/relation" - "github.com/openimsdk/tools/errs" ) //func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { @@ -38,7 +38,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*sdkws.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -51,8 +51,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return s.getFriend(ctx, req.UserID, ids) }, ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index b1b20dc40a..96f784fab0 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,10 +16,8 @@ package user import ( "context" - "errors" "math/rand" "strings" - "sync" "time" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" @@ -29,8 +27,6 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/open-im-server/v3/pkg/authverify" From a1523f47e7c8e762e2231df5e87af0c785c772cb Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 15:28:03 +0800 Subject: [PATCH 039/147] sync option --- config/redis.yml | 4 +-- internal/rpc/friend/sync.go | 26 -------------- internal/rpc/group/sync.go | 4 --- internal/rpc/incrversion/option.go | 45 ++++++++++++------------ pkg/common/storage/cache/friend.go | 2 +- pkg/common/storage/cache/group.go | 4 +-- pkg/common/storage/cache/redis/friend.go | 20 +++++------ pkg/common/storage/cache/redis/group.go | 42 +++++++++++----------- pkg/common/storage/controller/friend.go | 8 ++--- pkg/common/storage/controller/group.go | 19 +++++----- pkg/common/storage/model/friend.go | 2 -- 11 files changed, 72 insertions(+), 104 deletions(-) diff --git a/config/redis.yml b/config/redis.yml index 6fe0dd02d4..404d18953f 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,6 +1,6 @@ -address: [ localhost:16379 ] +address: [ 172.16.8.48:16379 ] username: '' password: openIM123 clusterMode: false db: 0 -maxRetry: 10 \ No newline at end of file +maxRetry: 10 diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 9af1c47d32..a73cf6a170 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -10,30 +10,6 @@ import ( "github.com/openimsdk/protocol/relation" ) -//func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { -// if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { -// return nil, err -// } -// if req.Keyword == "" { -// total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) -// if err != nil { -// return nil, err -// } -// return &pbfriend.SearchFriendsResp{ -// Total: total, -// Friends: friendsDB2PB(friends), -// }, nil -// } -// total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) -// if err != nil { -// return nil, err -// } -// return &pbfriend.SearchFriendsResp{ -// Total: total, -// Friends: friendsDB2PB(friends), -// }, nil -//} - func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err @@ -43,10 +19,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.FriendSyncCount, Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, - SortID: s.friendDatabase.FindSortFriendUserIDs, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) }, diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 6e0103c83f..b35816a7eb 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -15,10 +15,8 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou VersionKey: req.GroupID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.GroupSyncCount, Version: s.db.FindMemberIncrVersion, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, - SortID: s.db.FindSortGroupMemberUserIDs, Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, @@ -46,10 +44,8 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.GroupSyncCount, Version: s.db.FindJoinIncrVersion, CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, - SortID: s.db.FindSortJoinGroupIDs, Find: s.getGroupsInfo, ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index 3146985aa9..ceb024a3e1 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -16,6 +16,8 @@ import ( // return maxSync //} +const syncLimit = 200 + const ( tagQuery = iota + 1 tagFull @@ -23,17 +25,17 @@ const ( ) type Option[A, B any] struct { - Ctx context.Context - VersionKey string - VersionID string - VersionNumber uint64 - SyncLimit int + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + //SyncLimit int CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) - SortID func(ctx context.Context, dId string) ([]string, error) - Find func(ctx context.Context, ids []string) ([]A, error) - ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B + //SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B } func (o *Option[A, B]) newError(msg string) error { @@ -47,15 +49,15 @@ func (o *Option[A, B]) check() error { if o.VersionKey == "" { return o.newError("versionKey is empty") } - if o.SyncLimit <= 0 { - return o.newError("invalid synchronization quantity") - } + //if o.SyncLimit <= 0 { + // return o.newError("invalid synchronization quantity") + //} if o.Version == nil { return o.newError("func version is nil") } - if o.SortID == nil { - return o.newError("func allID is nil") - } + //if o.SortID == nil { + // return o.newError("func allID is nil") + //} if o.Find == nil { return o.newError("func find is nil") } @@ -81,7 +83,7 @@ func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { if o.CacheMaxVersion == nil { if o.validVersion() { *tag = tagQuery - return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) } *tag = tagFull return o.Version(o.Ctx, o.VersionKey, 0, 0) @@ -103,7 +105,7 @@ func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { return cache, nil } *tag = tagQuery - return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) } } @@ -131,12 +133,11 @@ func (o *Option[A, B]) Build() (*B, error) { deleteIDs []string changeIDs []string ) - //full := o.VersionID != version.ID.Hex() || version.Full() if full { - changeIDs, err = o.SortID(o.Ctx, o.VersionKey) - if err != nil { - return nil, err - } + //changeIDs, err = o.SortID(o.Ctx, o.VersionKey) + //if err != nil { + // return nil, err + //} } else { deleteIDs, changeIDs = version.DeleteAndChangeIDs() } diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 064e3baae7..b451d36757 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -39,7 +39,7 @@ type FriendCache interface { //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache - FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 00a46aaf82..73479bb1bb 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -59,8 +59,8 @@ type GroupCache interface { GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache - FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) - FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) DelMaxGroupMemberVersion(groupIDs ...string) GroupCache DelMaxJoinGroupVersion(userIDs ...string) GroupCache diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 62fad91cd1..01988310c9 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -193,16 +193,16 @@ func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.Fri return newFriendCache } -func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - userIDs, err := f.GetFriendIDs(ctx, ownerUserID) - if err != nil { - return nil, err - } - if len(userIDs) > f.syncCount { - userIDs = userIDs[:f.syncCount] - } - return userIDs, nil -} +//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// userIDs, err := f.GetFriendIDs(ctx, ownerUserID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > f.syncCount { +// userIDs = userIDs[:f.syncCount] +// } +// return userIDs, nil +//} func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 9699230efc..4015d5cd98 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -405,27 +405,27 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st }) } -func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { - userIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - if len(userIDs) > g.syncCount { - userIDs = userIDs[:g.syncCount] - } - return userIDs, nil -} - -func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { - groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) - if err != nil { - return nil, err - } - if len(groupIDs) > g.syncCount { - groupIDs = groupIDs[:g.syncCount] - } - return groupIDs, nil -} +//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := g.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > g.syncCount { +// userIDs = userIDs[:g.syncCount] +// } +// return userIDs, nil +//} +// +//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { +// groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) +// if err != nil { +// return nil, err +// } +// if len(groupIDs) > g.syncCount { +// groupIDs = groupIDs[:g.syncCount] +// } +// return groupIDs, nil +//} func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { keys := make([]string, 0, len(groupIDs)) diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index e03208df9e..42c2305988 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -78,7 +78,7 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) - FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) @@ -360,9 +360,9 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, }) } -func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) -} +//func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +//} func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f9ee65955d..14406420b0 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -110,9 +110,8 @@ type GroupDatabase interface { FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) - FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) - - FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) @@ -497,13 +496,13 @@ func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) } -func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { - return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) -} - -func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { - return g.cache.FindSortJoinGroupIDs(ctx, userID) -} +//func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { +// return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) +//} +// +//func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { +// return g.cache.FindSortJoinGroupIDs(ctx, userID) +//} func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { return g.cache.FindMaxGroupMemberVersion(ctx, groupID) diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index c7675e466d..60a40d9c2a 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -22,8 +22,6 @@ import ( type Friend struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` - FriendNickname string `bson:"friend_nickname"` - FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` From 58c4c13cf1179437f42a9a68c1ec84deaa5b604c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 7 Jun 2024 17:45:41 +0800 Subject: [PATCH 040/147] sync --- internal/api/group.go | 12 ++- internal/api/router.go | 2 + internal/rpc/friend/friend.go | 3 - internal/rpc/friend/sync.go | 14 ++-- internal/rpc/group/group.go | 5 +- internal/rpc/group/sync.go | 82 +++++++++++++++---- internal/rpc/incrversion/option.go | 40 ++++----- pkg/common/config/config.go | 6 +- pkg/common/storage/cache/redis/group.go | 3 - pkg/common/storage/controller/group.go | 13 ++- pkg/common/storage/database/group_member.go | 2 +- pkg/common/storage/database/mgo/friend.go | 8 +- .../storage/database/mgo/group_member.go | 18 ++-- .../storage/database/mgo/version_log.go | 18 ++-- pkg/common/storage/database/version_log.go | 2 +- pkg/common/storage/model/version_log.go | 26 ++++-- 16 files changed, 161 insertions(+), 93 deletions(-) diff --git a/internal/api/group.go b/internal/api/group.go index 5b8b9d0442..0bf61c7878 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -179,10 +179,18 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { return } resp.List[req.GroupID] = res - changeCount += len(res.Changes) + len(res.DeleteUserIds) - if changeCount >= int(res.SyncCount) { + changeCount += len(res.Insert) + len(res.Delete) + len(res.Update) + if changeCount >= 200 { break } } apiresp.GinSuccess(c, resp) } + +func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +} + +func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 445a24694c..69c026ffd6 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,6 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index cbc16cca96..12107125c1 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -64,9 +64,6 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - if config.RpcConfig.FriendSyncCount < 1 { - config.RpcConfig.FriendSyncCount = constant.MaxSyncPullNumber - } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a73cf6a170..cddd23f898 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -25,14 +25,14 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return s.getFriend(ctx, req.UserID, ids) }, ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { return &relation.GetIncrementalFriendsResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - DeleteUserIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, } }, } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 1cdcdb54b5..4d429b3d3b 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -77,9 +77,6 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - if config.RpcConfig.GroupSyncCount <= 0 { - config.RpcConfig.GroupSyncCount = constant.MaxSyncPullNumber - } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -104,7 +101,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) var gs groupServer - database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs), config.RpcConfig.GroupSyncCount) + database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database gs.user = userRpcClient gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index b35816a7eb..4bd34eb4d4 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,6 +2,9 @@ package group import ( "context" + "crypto/md5" + "encoding/binary" + "encoding/json" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -9,6 +12,57 @@ import ( "github.com/openimsdk/protocol/sdkws" ) +func (s *groupServer) idHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} + +func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { + vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) + if err != nil { + return nil, err + } + idHash := s.idHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { + vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := s.idHash(groupIDs) + if req.IdHash == idHash { + groupIDs = nil + } + return &pbgroup.GetIncrementalJoinGroupIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + GroupIDs: groupIDs, + }, nil +} + func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, @@ -21,14 +75,14 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), - DeleteUserIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, } }, } @@ -48,14 +102,14 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, Find: s.getGroupsInfo, ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), - DeleteGroupIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, } }, } diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index ceb024a3e1..f7a71244a0 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson/primitive" ) @@ -35,7 +34,7 @@ type Option[A, B any] struct { //SortID func(ctx context.Context, dId string) ([]string, error) Find func(ctx context.Context, ids []string) ([]A, error) ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B + Resp func(version *model.VersionLog, deleteIds []string, insertList, updateList []A, full bool) *B } func (o *Option[A, B]) newError(msg string) error { @@ -130,31 +129,28 @@ func (o *Option[A, B]) Build() (*B, error) { panic(fmt.Errorf("undefined tag %d", tag)) } var ( - deleteIDs []string - changeIDs []string + insertIds []string + deleteIds []string + updateIds []string ) - if full { - //changeIDs, err = o.SortID(o.Ctx, o.VersionKey) - //if err != nil { - // return nil, err - //} - } else { - deleteIDs, changeIDs = version.DeleteAndChangeIDs() + if !full { + insertIds, deleteIds, updateIds = version.DeleteAndChangeIDs() } - var list []A - if len(changeIDs) > 0 { - list, err = o.Find(o.Ctx, changeIDs) + var ( + insertList []A + updateList []A + ) + if len(insertIds) > 0 { + insertList, err = o.Find(o.Ctx, insertIds) if err != nil { return nil, err } - if (!full) && o.ID != nil && len(changeIDs) != len(list) { - foundIDs := datautil.SliceSetAny(list, o.ID) - for _, id := range changeIDs { - if _, ok := foundIDs[id]; !ok { - deleteIDs = append(deleteIDs, id) - } - } + } + if len(updateIds) > 0 { + updateList, err = o.Find(o.Ctx, updateIds) + if err != nil { + return nil, err } } - return o.Resp(version, deleteIDs, list, full), nil + return o.Resp(version, deleteIds, insertList, updateList, full), nil } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index a4177dc067..5313c196ac 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -244,8 +244,7 @@ type Friend struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - FriendSyncCount int `mapstructure:"friendSyncCount"` + Prometheus Prometheus `mapstructure:"prometheus"` } type Group struct { @@ -254,8 +253,7 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - GroupSyncCount int `mapstructure:"groupSyncCount"` + Prometheus Prometheus `mapstructure:"prometheus"` } type Msg struct { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 4015d5cd98..c4598567d3 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -45,7 +45,6 @@ type GroupCacheRedis struct { expireTime time.Duration rcClient *rockscache.Client groupHash cache.GroupHash - syncCount int } func NewGroupCacheRedis( @@ -56,7 +55,6 @@ func NewGroupCacheRedis( groupRequestDB database.GroupRequest, hashCode cache.GroupHash, opts *rockscache.Options, - syncCount int, ) cache.GroupCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Group.Topic}) g := localCache.Group @@ -70,7 +68,6 @@ func NewGroupCacheRedis( groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, groupHash: hashCode, - syncCount: syncCount, } } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 14406420b0..cb64bc73a3 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -117,6 +117,8 @@ type GroupDatabase interface { FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) + + FindJoinGroupID(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -127,14 +129,13 @@ func NewGroupDatabase( groupRequestDB database.GroupRequest, ctxTx tx.Tx, groupHash cache.GroupHash, - syncCount int, ) GroupDatabase { return &groupDatabase{ groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, ctxTx: ctxTx, - cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions(), syncCount), + cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()), } } @@ -146,6 +147,10 @@ type groupDatabase struct { cache cache.GroupCache } +func (g *groupDatabase) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) { + return g.cache.GetJoinedGroupIDs(ctx, userID) +} + func (g *groupDatabase) FindGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { return g.cache.GetGroupMembersInfo(ctx, groupID, userIDs) } @@ -243,7 +248,7 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma return err } for _, userID := range userIDs { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate); err != nil { return err } @@ -276,7 +281,7 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete } c = c.DelMaxJoinGroupVersion(userIDs...) if len(userIDs) > 0 { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, model.VersionStateDelete); err != nil { return err } } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index c397eda58a..0051d694f3 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -34,7 +34,7 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool - JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 646951d7d0..699d9cff6a 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -66,7 +66,7 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) } for ownerUserID, friendUserIDs := range mp { - if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false); err != nil { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil { return err } } @@ -83,7 +83,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, true) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete) }) } @@ -99,7 +99,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, false) + return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate) }) } @@ -189,7 +189,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index cb64c87a41..ece1d7941e 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -70,7 +70,7 @@ func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.Group gms[member.GroupID] = append(gms[member.GroupID], member.UserID) } for groupID, userIDs := range gms { - if err := g.member.IncrVersion(ctx, groupID, userIDs, false); err != nil { + if err := g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateInsert); err != nil { return err } } @@ -81,7 +81,7 @@ func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.Group gms[member.UserID] = append(gms[member.UserID], member.GroupID) } for userID, groupIDs := range gms { - if err := g.join.IncrVersion(ctx, userID, groupIDs, false); err != nil { + if err := g.join.IncrVersion(ctx, userID, groupIDs, model.VersionStateInsert); err != nil { return err } } @@ -97,12 +97,12 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s return mongoutil.IncrVersion(func() error { return mongoutil.DeleteMany(ctx, g.coll, filter) }, func() error { - return g.member.IncrVersion(ctx, groupID, userIDs, true) + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }, func() error { if len(userIDs) == 0 { return nil } - return g.member.IncrVersion(ctx, groupID, userIDs, true) + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }) } @@ -110,9 +110,9 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return mongoutil.IncrVersion(func() error { return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, true) + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }, func() error { - return g.join.IncrVersion(ctx, groupID, []string{userID}, true) + return g.join.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }) } @@ -123,7 +123,7 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, false) + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }) } @@ -174,8 +174,8 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { return ok } -func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error { - return g.join.IncrVersion(ctx, userID, groupIDs, deleted) +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error { + return g.join.IncrVersion(ctx, userID, groupIDs, state) } func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 5629c5c00f..8ab11007d1 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -36,7 +36,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { return err } -func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } @@ -44,19 +44,19 @@ func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []stri return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) } now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + res, err := l.writeLogBatch(ctx, dId, eIds, state, now) if err != nil { return err } if res.MatchedCount > 0 { return nil } - if _, err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + if _, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { return nil } else if !mongo.IsDuplicateKeyError(err) { return err } - if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + if res, err := l.writeLogBatch(ctx, dId, eIds, state, now); err != nil { return err } else if res.MatchedCount == 0 { return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) @@ -64,7 +64,7 @@ func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []stri return nil } -func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*model.VersionLogTable, error) { +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLogTable, error) { wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, @@ -76,7 +76,7 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, for _, eId := range eIds { wl.Logs = append(wl.Logs, model.VersionLogElem{ EID: eId, - Deleted: deleted, + State: state, Version: database.FirstVersion, LastUpdate: now, }) @@ -85,7 +85,7 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, return &wl, err } -func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*mongo.UpdateResult, error) { if eIds == nil { eIds = []string{} } @@ -97,7 +97,7 @@ func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []st elems = append(elems, bson.M{ "e_id": eId, "version": "$version", - "deleted": deleted, + "state": state, "last_update": now, }) } @@ -159,7 +159,7 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } - if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { + if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { return res.VersionLog(), nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index 3450d2776e..c9dc09540b 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -12,7 +12,7 @@ const ( ) type VersionLog interface { - IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error + IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index a09f493a86..e1b5fe7c52 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -1,13 +1,22 @@ package model import ( + "context" + "errors" + "github.com/openimsdk/tools/log" "go.mongodb.org/mongo-driver/bson/primitive" "time" ) +const ( + VersionStateInsert = iota + 1 + VersionStateDelete + VersionStateUpdate +) + type VersionLogElem struct { EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` + State int32 `bson:"state"` Version uint `bson:"version"` LastUpdate time.Time `bson:"last_update"` } @@ -43,12 +52,17 @@ type VersionLog struct { LogLen int `bson:"log_len"` } -func (v *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { +func (v *VersionLog) DeleteAndChangeIDs() (insertIds, deleteIds, updateIds []string) { for _, l := range v.Logs { - if l.Deleted { - delIds = append(delIds, l.EID) - } else { - changeIds = append(changeIds, l.EID) + switch l.State { + case VersionStateInsert: + insertIds = append(insertIds, l.EID) + case VersionStateDelete: + deleteIds = append(deleteIds, l.EID) + case VersionStateUpdate: + updateIds = append(updateIds, l.EID) + default: + log.ZError(context.Background(), "invalid version status found", errors.New("dirty database data"), "objID", v.ID.Hex(), "did", v.DID, "elem", l) } } return From caebdf32caf3e9eb051cf49e8cf19cde3f1aecfe Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 11 Jun 2024 18:38:36 +0800 Subject: [PATCH 041/147] sync --- internal/rpc/group/notification.go | 22 +++++++ pkg/common/cmd/group.go | 3 +- pkg/common/storage/database/mgo/black.go | 2 +- .../storage/database/mgo/conversation.go | 3 +- pkg/common/storage/database/mgo/friend.go | 4 +- .../storage/database/mgo/friend_request.go | 2 +- pkg/common/storage/database/mgo/group.go | 2 +- .../storage/database/mgo/group_member.go | 6 +- .../storage/database/mgo/group_request.go | 2 +- pkg/common/storage/database/mgo/log.go | 2 +- pkg/common/storage/database/mgo/object.go | 2 +- pkg/common/storage/database/mgo/user.go | 2 +- .../storage/database/mgo/version_log.go | 62 +++++++++++-------- .../storage/database/mgo/version_test.go | 39 ++++++++++++ pkg/common/storage/database/name.go | 17 +++++ pkg/common/storage/model/version_log.go | 2 +- pkg/common/storage/versionctx/rpc.go | 14 +++++ pkg/common/storage/versionctx/version.go | 48 ++++++++++++++ 18 files changed, 193 insertions(+), 41 deletions(-) create mode 100644 pkg/common/storage/database/mgo/version_test.go create mode 100644 pkg/common/storage/database/name.go create mode 100644 pkg/common/storage/versionctx/rpc.go create mode 100644 pkg/common/storage/versionctx/version.go diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index cfa62c85db..13e593c71a 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -17,7 +17,9 @@ package group import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -287,6 +289,15 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, collName string, id string) { + for _, coll := range versionctx.GetVersionLog(ctx).Get() { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + return + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -297,6 +308,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -380,6 +392,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -467,6 +480,7 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -480,6 +494,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } @@ -503,6 +518,7 @@ func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, } tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) } @@ -524,6 +540,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return } tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) } @@ -564,6 +581,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -588,6 +606,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -666,6 +685,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -689,6 +709,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -713,5 +734,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f158b8c626..20124be957 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/rpc/group" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -58,5 +59,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start) + a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/storage/database/mgo/black.go b/pkg/common/storage/database/mgo/black.go index cf74cfab14..4a7a35e6f1 100644 --- a/pkg/common/storage/database/mgo/black.go +++ b/pkg/common/storage/database/mgo/black.go @@ -27,7 +27,7 @@ import ( ) func NewBlackMongo(db *mongo.Database) (database.Black, error) { - coll := db.Collection("black") + coll := db.Collection(database.BlackName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 9c35f841b6..b462d39583 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" @@ -29,7 +30,7 @@ import ( ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { - coll := db.Collection("conversation") + coll := db.Collection(database.ConversationName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 699d9cff6a..18d80d47d0 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -34,7 +34,7 @@ type FriendMgo struct { // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. func NewFriendMongo(db *mongo.Database) (database.Friend, error) { - coll := db.Collection("friend") + coll := db.Collection(database.FriendName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, @@ -45,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := NewVersionLog(db.Collection("friend_version")) + owner, err := NewVersionLog(db.Collection(database.FriendVersionName)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 0d60b213dd..4eed2f4a26 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -27,7 +27,7 @@ import ( ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { - coll := db.Collection("friend_request") + coll := db.Collection(database.FriendRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "from_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 630bc02919..3be7883af9 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -30,7 +30,7 @@ import ( ) func NewGroupMongo(db *mongo.Database) (database.Group, error) { - coll := db.Collection("group") + coll := db.Collection(database.GroupName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ece1d7941e..a9ac262ab1 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -29,7 +29,7 @@ import ( ) func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { - coll := db.Collection("group_member") + coll := db.Collection(database.GroupMemberName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -40,11 +40,11 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - member, err := NewVersionLog(db.Collection("group_member_version")) + member, err := NewVersionLog(db.Collection(database.GroupMemberVersionName)) if err != nil { return nil, err } - join, err := NewVersionLog(db.Collection("group_join_version")) + join, err := NewVersionLog(db.Collection(database.GroupJoinVersionName)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index 4ae7785276..b1942b7083 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -28,7 +28,7 @@ import ( ) func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { - coll := db.Collection("group_request") + coll := db.Collection(database.GroupRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/log.go b/pkg/common/storage/database/mgo/log.go index 51715bd771..6ff4c60395 100644 --- a/pkg/common/storage/database/mgo/log.go +++ b/pkg/common/storage/database/mgo/log.go @@ -28,7 +28,7 @@ import ( ) func NewLogMongo(db *mongo.Database) (database.Log, error) { - coll := db.Collection("log") + coll := db.Collection(database.LogName) _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ { Keys: bson.D{ diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 8ed7b3a561..df4d10ec4f 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -27,7 +27,7 @@ import ( ) func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { - coll := db.Collection("s3") + coll := db.Collection(database.ObjectName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "name", Value: 1}, diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 793b8cdc85..8978e64ebf 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -31,7 +31,7 @@ import ( ) func NewUserMongo(db *mongo.Database) (database.User, error) { - coll := db.Collection("user") + coll := db.Collection(database.UserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 8ab11007d1..b53057fe4b 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -5,9 +5,9 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -37,34 +37,41 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { } func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { - if len(eIds) == 0 { - return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + _, err := l.IncrVersionResult(ctx, dId, eIds, state) + return err +} + +func (l *VersionLogMgo) IncrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + vl, err := l.incrVersionResult(ctx, dId, eIds, state) + if err != nil { + return nil, err } - if datautil.Duplicate(eIds) { - return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + versionctx.GetVersionLog(ctx).Append(versionctx.Collection{ + Name: l.coll.Name(), + Doc: vl, + }) + return vl, nil +} + +func (l *VersionLogMgo) incrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, state, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil + if res, err := l.writeLogBatch2(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err } - if _, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { - return nil + if res, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { + return res, nil } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLogBatch(ctx, dId, eIds, state, now); err != nil { - return err - } else if res.MatchedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + return nil, err } - return nil + return l.writeLogBatch2(ctx, dId, eIds, state, now) } -func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLogTable, error) { +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, @@ -81,11 +88,13 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, LastUpdate: now, }) } - _, err := l.coll.InsertOne(ctx, &wl) - return &wl, err + if _, err := l.coll.InsertOne(ctx, &wl); err != nil { + return nil, err + } + return wl.VersionLog(), nil } -func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { if eIds == nil { eIds = []string{} } @@ -142,7 +151,8 @@ func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []st "$unset": "delete_e_ids", }, } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0}) + return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) } func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { @@ -160,7 +170,7 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u return nil, err } if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { - return res.VersionLog(), nil + return res, nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) } else { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go new file mode 100644 index 0000000000..236c61a2ce --- /dev/null +++ b/pkg/common/storage/database/mgo/version_test.go @@ -0,0 +1,39 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("version_test") + tmp, err := NewVersionLog(coll) + if err != nil { + panic(err) + } + vl := tmp.(*VersionLogMgo) + res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) + if err != nil { + t.Log(err) + return + } + t.Logf("%+v", res) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go new file mode 100644 index 0000000000..986f22a1a2 --- /dev/null +++ b/pkg/common/storage/database/name.go @@ -0,0 +1,17 @@ +package database + +const ( + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" +) diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index e1b5fe7c52..11a40ef24f 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -38,7 +38,7 @@ func (v *VersionLogTable) VersionLog() *VersionLog { Version: v.Version, Deleted: v.Deleted, LastUpdate: v.LastUpdate, - LogLen: 0, + LogLen: len(v.Logs), } } diff --git a/pkg/common/storage/versionctx/rpc.go b/pkg/common/storage/versionctx/rpc.go new file mode 100644 index 0000000000..67b95aebd5 --- /dev/null +++ b/pkg/common/storage/versionctx/rpc.go @@ -0,0 +1,14 @@ +package versionctx + +import ( + "context" + "google.golang.org/grpc" +) + +func EnableVersionCtx() grpc.ServerOption { + return grpc.ChainUnaryInterceptor(enableVersionCtxInterceptor) +} + +func enableVersionCtxInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(WithVersionLog(ctx), req) +} diff --git a/pkg/common/storage/versionctx/version.go b/pkg/common/storage/versionctx/version.go new file mode 100644 index 0000000000..5db8856401 --- /dev/null +++ b/pkg/common/storage/versionctx/version.go @@ -0,0 +1,48 @@ +package versionctx + +import ( + "context" + tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "sync" +) + +type Collection struct { + Name string + Doc *tablerelation.VersionLog +} + +type versionKey struct{} + +func WithVersionLog(ctx context.Context) context.Context { + return context.WithValue(ctx, versionKey{}, &VersionLog{}) +} + +func GetVersionLog(ctx context.Context) *VersionLog { + if v, ok := ctx.Value(versionKey{}).(*VersionLog); ok { + return v + } + return nil +} + +type VersionLog struct { + lock sync.Mutex + data []Collection +} + +func (v *VersionLog) Append(data ...Collection) { + if v == nil || len(data) == 0 { + return + } + v.lock.Lock() + defer v.lock.Unlock() + v.data = append(v.data, data...) +} + +func (v *VersionLog) Get() []Collection { + if v == nil { + return nil + } + v.lock.Lock() + defer v.lock.Unlock() + return v.data +} From 1b1027192dc7e6c3ff6d1b27373e743dc0689ae6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 13 Jun 2024 09:45:14 +0800 Subject: [PATCH 042/147] go.mod --- go.mod | 6 +++--- go.sum | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 3ef42ef80f..4f4cd4b686 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.4 - github.com/openimsdk/tools v0.0.49-alpha.24 + github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -177,4 +177,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 835bce361d..5a72e28f2c 100644 --- a/go.sum +++ b/go.sum @@ -270,12 +270,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= -github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/protocol v0.0.69-alpha.4 h1:QJkOFV5Hlu7CbkHG5smeVw+5fx5DVkpNJWqlAOJxuIY= -github.com/openimsdk/protocol v0.0.69-alpha.4/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= -github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= +github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= +github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From ef71d0cf6247b386964dfdd2a6a8b4206cb759af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 14 Jun 2024 10:54:24 +0800 Subject: [PATCH 043/147] seq --- pkg/common/storage/cache/cachekey/seq1.go | 19 ++ pkg/common/storage/cache/redis/seq1.go.txt | 246 +++++++++++++++++++++ pkg/common/storage/controller/msg.go | 42 ++-- pkg/common/storage/database/mgo/seq.go | 75 +++++++ pkg/common/storage/database/name.go | 1 + pkg/common/storage/database/seq.go | 10 + pkg/common/storage/model/seq.go | 7 + pkg/msgprocessor/conversation.go | 4 + 8 files changed, 384 insertions(+), 20 deletions(-) create mode 100644 pkg/common/storage/cache/cachekey/seq1.go create mode 100644 pkg/common/storage/cache/redis/seq1.go.txt create mode 100644 pkg/common/storage/database/mgo/seq.go create mode 100644 pkg/common/storage/database/seq.go create mode 100644 pkg/common/storage/model/seq.go diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go new file mode 100644 index 0000000000..df3086d326 --- /dev/null +++ b/pkg/common/storage/cache/cachekey/seq1.go @@ -0,0 +1,19 @@ +package cachekey + +const ( + MallocSeq = "MALLOC_SEQ:" + MallocSeqLock = "MALLOC_SEQ_LOCK:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" +) + +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocSeqLockKey(conversationID string) string { + return MallocSeqLock + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID +} diff --git a/pkg/common/storage/cache/redis/seq1.go.txt b/pkg/common/storage/cache/redis/seq1.go.txt new file mode 100644 index 0000000000..9ccb83e668 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq1.go.txt @@ -0,0 +1,246 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "github.com/dtm-labs/rockscache" + "github.com/google/uuid" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +var errLock = errors.New("lock failed") + +type MallocSeq interface { + Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} + +func NewSeqCache1(rdb redis.UniversalClient, mgo database.Seq) MallocSeq { + opt := rockscache.NewDefaultOptions() + opt.EmptyExpire = time.Second * 3 + opt.Delay = time.Second / 2 + return &seqCache1{ + rdb: rdb, + mgo: mgo, + rocks: rockscache.NewClient(rdb, opt), + lockExpire: time.Minute * 1, + seqExpire: time.Hour * 24 * 7, + minSeqExpire: time.Hour * 1, + groupMinNum: 1000, + userMinNum: 100, + } +} + +type seqCache1 struct { + rdb redis.UniversalClient + rocks *rockscache.Client + mgo database.Seq + lockExpire time.Duration + seqExpire time.Duration + minSeqExpire time.Duration + groupMinNum int64 + userMinNum int64 +} + +/* +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +*/ + +func (s *seqCache1) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + for i := 0; i < 10; i++ { + res, err := s.rdb.LIndex(ctx, cachekey.GetMallocSeqKey(conversationID), 0).Int64() + if err == nil { + return res, nil + } else if !errors.Is(err, redis.Nil) { + return 0, errs.Wrap(err) + } + + if err := s.mallocSeq(ctx, conversationID, 0, nil); err != nil { + return 0, err + } + } + return 0, errs.New("get max seq timeout") +} + +func (s *seqCache1) unlock(ctx context.Context, key string, owner string) error { + script := ` +local value = redis.call("GET", KEYS[1]) +if value == false then + return 0 +end +if value == ARGV[1] then + redis.call("DEL", KEYS[1]) + return 1 +end +return 2 +` + state, err := s.rdb.Eval(ctx, script, []string{key}, owner).Int() + if err != nil { + return errs.Wrap(err) + } + switch state { + case 0: + return errs.Wrap(redis.Nil) + case 1: + return nil + case 2: + return errs.New("not the lock holder") + default: + return errs.New(fmt.Sprintf("unknown state: %d", state)) + } +} + +func (s *seqCache1) initMallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { + owner := uuid.New().String() + ok, err := s.rdb.SetNX(ctx, cachekey.GetMallocSeqLockKey(conversationID), owner, s.lockExpire).Result() + if err != nil { + return nil, err + } + + seq, err := s.mgo.Malloc(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := seq - size + 1; i <= seq; i++ { + seqs = append(seqs, i) + } + return seqs, nil +} + +func (s *seqCache1) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache[int64](ctx, s.rocks, cachekey.GetMallocMinSeqKey(conversationID), s.minSeqExpire, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) +} + +func (s *seqCache1) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + return s.deleteMinSeqCache(ctx, conversationID) +} + +func (s *seqCache1) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { + if size <= 0 { + return nil, errs.Wrap(errors.New("size must be greater than 0")) + } + seqKey := cachekey.GetMallocSeqKey(conversationID) + lockKey := cachekey.GetMallocSeqLockKey(conversationID) + for i := 0; i < 10; i++ { + seqs, err := s.lpop(ctx, seqKey, lockKey, size) + if err != nil { + return nil, err + } + if len(seqs) < int(size) { + if err := s.mallocSeq(ctx, conversationID, size, &seqs); err != nil { + return nil, err + } + } + if len(seqs) >= int(size) { + return seqs, nil + } + } + return nil, errs.ErrInternalServer.WrapMsg("malloc seq failed") +} + +func (s *seqCache1) push(ctx context.Context, seqKey string, seqs []int64) error { + script := ` +redis.call("DEL", KEYS[1]) +for i = 2, #ARGV do + redis.call("RPUSH", KEYS[1], ARGV[i]) +end +redis.call("EXPIRE", KEYS[1], ARGV[1]) +return 1 +` + argv := make([]any, 0, 1+len(seqs)) + argv = append(argv, s.seqExpire.Seconds()) + for _, seq := range seqs { + argv = append(argv, seq) + } + err := s.rdb.Eval(ctx, script, []string{seqKey}, argv...).Err() + return errs.Wrap(err) +} + +func (s *seqCache1) lpop(ctx context.Context, seqKey, lockKey string, size int64) ([]int64, error) { + script := ` +local result = redis.call("LRANGE", KEYS[1], 0, ARGV[1]-1) +if #result == 0 then + return result +end +redis.call("LTRIM", KEYS[1], #result, -1) +if redis.call("LLEN", KEYS[1]) == 0 then + redis.call("DEL", KEYS[2]) +end +return result +` + res, err := s.rdb.Eval(ctx, script, []string{seqKey, lockKey}, size).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return res, nil +} + +func (s *seqCache1) getMongoStepSize(conversationID string, size int64) int64 { + var num int64 + if msgprocessor.IsGroupConversationID(conversationID) { + num = s.groupMinNum + } else { + num = s.userMinNum + } + if size > num { + num += size + } + return num +} + +func (s *seqCache1) mallocSeq(ctx context.Context, conversationID string, size int64, seqs *[]int64) error { + var delMinSeqKey bool + _, err := getCache[string](ctx, s.rocks, cachekey.GetMallocSeqLockKey(conversationID), s.lockExpire, func(ctx context.Context) (string, error) { + res, err := s.mgo.Malloc(ctx, conversationID, s.getMongoStepSize(conversationID, size)) + if err != nil { + return "", err + } + delMinSeqKey = res[0] == 1 + if seqs != nil && size > 0 { + if len(*seqs) > 0 && (*seqs)[len(*seqs)-1]+1 == res[0] { + n := size - int64(len(*seqs)) + *seqs = append(*seqs, res[:n]...) + res = res[n:] + } else { + *seqs = res[:size] + res = res[size:] + } + } + if err := s.push(ctx, cachekey.GetMallocSeqKey(conversationID), res); err != nil { + return "", err + } + return strconv.Itoa(int(time.Now().UnixMicro())), nil + }) + if delMinSeqKey { + s.deleteMinSeqCache(ctx, conversationID) + } + return err +} + +func (s *seqCache1) deleteMinSeqCache(ctx context.Context, conversationID string) error { + return s.rocks.TagAsDeleted2(ctx, cachekey.GetMallocMinSeqKey(conversationID)) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8eb9e8e6fd..f0efe5efdf 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -69,26 +69,26 @@ type CommonMsgDatabase interface { DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, seqs []int64) error - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error + //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) + //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMinSeq(ctx context.Context, conversationID string) (int64, error) + //GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + //GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) + //SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error + //SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) + //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) + //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) @@ -349,6 +349,8 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { + + // TODO set SEQ currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) if err != nil && errs.Unwrap(err) != redis.Nil { log.ZError(ctx, "storage.seq.GetMaxSeq", err) @@ -817,9 +819,9 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u } } -func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) -} +//func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { +// return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) +//} func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { return db.seq.GetMaxSeqs(ctx, conversationIDs) @@ -837,13 +839,13 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int return db.seq.SetMinSeqs(ctx, seqs) } -func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMinSeqs(ctx, conversationIDs) -} - -func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMinSeq(ctx, conversationID) -} +//func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { +// return db.seq.GetMinSeqs(ctx, conversationIDs) +//} +// +//func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +// return db.seq.GetMinSeq(ctx, conversationID) +//} func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq.go new file mode 100644 index 0000000000..8592bf63cd --- /dev/null +++ b/pkg/common/storage/database/mgo/seq.go @@ -0,0 +1,75 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqMongo(db *mongo.Database) (*SeqMongo, error) { + coll := db.Collection("seq") + return &SeqMongo{coll: coll}, nil +} + +type SeqMongo struct { + coll *mongo.Collection +} + +func (s *SeqMongo) MallocSeq(ctx context.Context, conversationID string, size int64) (int64, error) { + if size <= 0 { + return 0, errors.New("size must be greater than 0") + } + filter := map[string]any{"conversation_id": conversationID} + update := map[string]any{ + "$inc": map[string]any{"max_seq": size}, + "$set": map[string]any{"min_seq": 1}, + } + opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) + return mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) +} + +func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { + seq, err := s.MallocSeq(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := seq - size + 1; i <= seq; i++ { + seqs = append(seqs, i) + } + return seqs, nil +} + +func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *SeqMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) +} + +func (s *SeqMongo) GetConversation(ctx context.Context, conversationID string) (*model.Seq, error) { + return mongoutil.FindOne[*model.Seq](ctx, s.coll, bson.M{"conversation_id": conversationID}) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 986f22a1a2..f496d4b41a 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -14,4 +14,5 @@ const ( LogName = "log" ObjectName = "s3" UserName = "user" + SeqName = "seq" ) diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go new file mode 100644 index 0000000000..869b607a5b --- /dev/null +++ b/pkg/common/storage/database/seq.go @@ -0,0 +1,10 @@ +package database + +import "context" + +type Seq interface { + Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go new file mode 100644 index 0000000000..3db0091676 --- /dev/null +++ b/pkg/common/storage/model/seq.go @@ -0,0 +1,7 @@ +package model + +type Seq struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} diff --git a/pkg/msgprocessor/conversation.go b/pkg/msgprocessor/conversation.go index b369269cc8..f8140cc7df 100644 --- a/pkg/msgprocessor/conversation.go +++ b/pkg/msgprocessor/conversation.go @@ -24,6 +24,10 @@ import ( "google.golang.org/protobuf/proto" ) +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "g_") || strings.HasPrefix(conversationID, "sg_") +} + func GetNotificationConversationIDByMsg(msg *sdkws.MsgData) string { switch msg.SessionType { case constant.SingleChatType: From 904842b15da197711bfa7fdb27cb74445ff8b6ee Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:04:03 +0800 Subject: [PATCH 044/147] update: go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4f4cd4b686..28e51cdaae 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/protocol v0.0.69-alpha.10 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5a72e28f2c..f29079ec3e 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= -github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.10 h1:reuNuQt77a57Sd8mUgy8PqTA+sG6rk6CfzHkE9DvBto= +github.com/openimsdk/protocol v0.0.69-alpha.10/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From c5fa596cdd3a8905a35dc215de86f7ea9ac8ad2b Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:14:43 +0800 Subject: [PATCH 045/147] refactor: change incremental to full --- internal/api/group.go | 8 ++++---- internal/api/router.go | 4 ++-- internal/rpc/group/sync.go | 25 +++++++------------------ pkg/util/hashutil/id.go | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 pkg/util/hashutil/id.go diff --git a/internal/api/group.go b/internal/api/group.go index 0bf61c7878..91992004c5 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -187,10 +187,10 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { apiresp.GinSuccess(c, resp) } -func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) } -func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 69c026ffd6..e0ec652d50 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,8 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) - groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) + groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 4bd34eb4d4..bd6fc6399e 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,26 +2,15 @@ package group import ( "context" - "crypto/md5" - "encoding/binary" - "encoding/json" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" ) -func (s *groupServer) idHash(ids []string) uint64 { - if len(ids) == 0 { - return 0 - } - data, _ := json.Marshal(ids) - sum := md5.Sum(data) - return binary.BigEndian.Uint64(sum[:]) -} - -func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err @@ -30,11 +19,11 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req if err != nil { return nil, err } - idHash := s.idHash(userIDs) + idHash := hashutil.IdHash(userIDs) if req.IdHash == idHash { userIDs = nil } - return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + return &pbgroup.GetFullGroupMemberUserIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, @@ -42,7 +31,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req }, nil } -func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -51,11 +40,11 @@ func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgro if err != nil { return nil, err } - idHash := s.idHash(groupIDs) + idHash := hashutil.IdHash(groupIDs) if req.IdHash == idHash { groupIDs = nil } - return &pbgroup.GetIncrementalJoinGroupIDsResp{ + return &pbgroup.GetFullJoinGroupIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, diff --git a/pkg/util/hashutil/id.go b/pkg/util/hashutil/id.go new file mode 100644 index 0000000000..52e7f4c6f2 --- /dev/null +++ b/pkg/util/hashutil/id.go @@ -0,0 +1,16 @@ +package hashutil + +import ( + "crypto/md5" + "encoding/binary" + "encoding/json" +) + +func IdHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} From f19f6f9112a94bf0eef831f902de573ab06e6297 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:17:59 +0800 Subject: [PATCH 046/147] feat: get full friend user ids --- internal/rpc/friend/friend.go | 42 +++++++++++++++++------------------ internal/rpc/friend/sync.go | 26 ++++++++++++++++++++-- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 12107125c1..622e19f422 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -41,7 +41,7 @@ import ( ) type friendServer struct { - friendDatabase controller.FriendDatabase + db controller.FriendDatabase blackDatabase controller.BlackDatabase userRpcClient *rpcclient.UserRpcClient notificationSender *FriendNotificationSender @@ -102,7 +102,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Register Friend server with refactored MongoDB and Redis integrations relation.RegisterFriendServer(server, &friendServer{ - friendDatabase: controller.NewFriendDatabase( + db: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB, redis.GetRocksCacheOptions()), @@ -139,14 +139,14 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.Apply return nil, err } - in1, in2, err := s.friendDatabase.CheckIn(ctx, req.FromUserID, req.ToUserID) + in1, in2, err := s.db.CheckIn(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } if in1 && in2 { return nil, servererrs.ErrRelationshipAlready.WrapMsg("already friends has f") } - if err = s.friendDatabase.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { + if err = s.db.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { return nil, err } s.notificationSender.FriendApplicationAddNotification(ctx, req) @@ -173,7 +173,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr return nil, err } - if err := s.friendDatabase.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { + if err := s.db.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { return nil, err } for _, userID := range req.FriendUserIDs { @@ -205,7 +205,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res if err := s.webhookBeforeAddFriendAgree(ctx, &s.config.WebhooksConfig.BeforeAddFriendAgree, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - err := s.friendDatabase.AgreeFriendRequest(ctx, &friendRequest) + err := s.db.AgreeFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -213,7 +213,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { - err := s.friendDatabase.RefuseFriendRequest(ctx, &friendRequest) + err := s.db.RefuseFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -229,11 +229,11 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFri if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { + if err := s.db.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { return nil, err } s.notificationSender.FriendDeletedNotification(ctx, req) @@ -250,11 +250,11 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { + if err := s.db.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { return nil, err } s.webhookAfterSetFriendRemark(ctx, &s.config.WebhooksConfig.AfterSetFriendRemark, req) @@ -281,7 +281,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend if len(friendUserIDs) == 0 { return nil, nil } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + friends, err := s.db.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) if err != nil { return nil, err } @@ -292,7 +292,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, req *relation.GetDesignatedFriendsApplyReq, ) (resp *relation.GetDesignatedFriendsApplyResp, err error) { - friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) + friendRequests, err := s.db.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } @@ -309,7 +309,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -327,7 +327,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -342,7 +342,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r // ok. func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { resp = &relation.IsFriendResp{} - resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) + resp.InUser1Friends, resp.InUser2Friends, err = s.db.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } @@ -353,7 +353,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.G if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + total, friends, err := s.db.PageOwnerFriends(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -371,7 +371,7 @@ func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriend return nil, err } resp = &relation.GetFriendIDsResp{} - resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) + resp.FriendIDs, err = s.db.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } @@ -389,7 +389,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio if err != nil { return nil, err } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) if err != nil { return nil, err } @@ -453,7 +453,7 @@ func (s *friendServer) UpdateFriends( return nil, errs.ErrArgs.WrapMsg("friendIDList repeated") } - _, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + _, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } @@ -469,7 +469,7 @@ func (s *friendServer) UpdateFriends( if req.Ex != nil { val["ex"] = req.Ex.Value } - if err = s.friendDatabase.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { + if err = s.db.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { return nil, err } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index cddd23f898..faaa987f20 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,6 +2,7 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" @@ -10,6 +11,27 @@ import ( "github.com/openimsdk/protocol/relation" ) +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &relation.GetFullFriendUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err @@ -19,8 +41,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - Version: s.friendDatabase.FindFriendIncrVersion, - CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, + Version: s.db.FindFriendIncrVersion, + CacheMaxVersion: s.db.FindMaxFriendVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) }, From cee1a496e970cb2f4b64617d102e892ceafd4537 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 18:29:54 +0800 Subject: [PATCH 047/147] feat: api and config --- config/redis.yml | 2 +- internal/api/friend.go | 4 ++++ internal/api/router.go | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/redis.yml b/config/redis.yml index 404d18953f..87abed0e1c 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ 172.16.8.48:16379 ] +address: [ localhost:16379 ] username: '' password: openIM123 clusterMode: false diff --git a/internal/api/friend.go b/internal/api/friend.go index 6912fdbbbc..9266fe75ca 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -102,3 +102,7 @@ func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } + +func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { + a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index e0ec652d50..0f46f26baf 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -89,6 +89,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) + friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") From fe4842b49690477e38aa0a0c534dcb3d4f4e443f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 14 Jun 2024 18:49:05 +0800 Subject: [PATCH 048/147] seq --- pkg/common/storage/cache/redis/seq1.go | 253 ++++++++++++++++++ pkg/common/storage/cache/redis/seq_test.go | 27 ++ pkg/common/storage/database/mgo/seq.go | 21 +- .../mgo/{version_test.go => seq_test.go} | 18 +- pkg/common/storage/database/seq.go | 2 +- 5 files changed, 298 insertions(+), 23 deletions(-) create mode 100644 pkg/common/storage/cache/redis/seq1.go create mode 100644 pkg/common/storage/cache/redis/seq_test.go rename pkg/common/storage/database/mgo/{version_test.go => seq_test.go} (58%) diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq1.go new file mode 100644 index 0000000000..6374f53ccf --- /dev/null +++ b/pkg/common/storage/cache/redis/seq1.go @@ -0,0 +1,253 @@ +package redis + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +type RedisHash struct { + NextSeq int64 + LastSeq int64 +} + +func NewTestSeq() *SeqMalloc { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &SeqMalloc{ + rdb: rdb, + mgo: model, + //lockTime: time.Second * 30, + lockTime: time.Second * 60 * 60 * 24 * 1, + dataTime: time.Second * 60 * 60 * 24 * 7, + } +} + +type SeqMalloc struct { + rdb redis.UniversalClient + mgo database.Seq + lockTime time.Duration + dataTime time.Duration +} + +func (s *SeqMalloc) getSeqMallocKey(conversationID string) string { + return cachekey.GetMallocSeqKey(conversationID) +} + +func (s *SeqMalloc) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { + if lastSeq < currSeq { + return 0, errs.New("lastSeq must be greater than currSeq") + } + // 0： 成功 + // 1： 成功 锁过期，但未被其他人锁 + // 2： 已经被锁，但是锁的不是自己 + script := ` +local key = KEYS[1] +local lockValue = ARGV[1] +local dataSecond = ARGV[2] +local curr_seq = tonumber(ARGV[3]) +local last_seq = tonumber(ARGV[4]) +if redis.call("EXISTS", key) == 0 then + redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) + redis.call("EXPIRE", key, dataSecond) + return 1 +end +if redis.call("HGET", key, "LOCK") ~= lockValue then + return 2 +end +redis.call("HDEL", key, "LOCK") +redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) +redis.call("EXPIRE", key, dataSecond) +return 0 +` + result, err := s.rdb.Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq).Int64() + if err != nil { + return 0, errs.Wrap(err) + } + return result, nil +} + +// malloc size=0为获取当前seq size>0为分配seq +func (s *SeqMalloc) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0： 成功 + // 1： 需要获取，并加锁 + // 2： 已经被锁 + // 3： 超过最大值，并加锁 + script := ` +local key = KEYS[1] +local size = tonumber(ARGV[1]) +local lockSecond = ARGV[2] +local dataSecond = ARGV[3] +local result = {} +if redis.call("EXISTS", key) == 0 then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 1) + table.insert(result, lockValue) + return result +end +if redis.call("HEXISTS", key, "LOCK") == 1 then + table.insert(result, 2) + return result +end +local curr_seq = tonumber(redis.call("HGET", key, "CURR")) +local last_seq = tonumber(redis.call("HGET", key, "LAST")) +if size == 0 then + table.insert(result, 0) + table.insert(result, curr_seq) + table.insert(result, last_seq) + return result +end +local max_seq = curr_seq + size +if max_seq > last_seq then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("HSET", key, "CURR", last_seq) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 3) + table.insert(result, curr_seq) + table.insert(result, last_seq) + table.insert(result, lockValue) + return result +end +redis.call("HSET", key, "CURR", max_seq) +table.insert(result, 0) +table.insert(result, curr_seq) +table.insert(result, last_seq) +return result +` + result, err := s.rdb.Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second)).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return result, nil +} + +func (s *SeqMalloc) wait(ctx context.Context) error { + timer := time.NewTimer(time.Second / 4) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { + for i := 0; i < 10; i++ { + state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) + if err != nil { + log.ZError(ctx, "set seq cache failed", err, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq, "count", i+1) + if err := s.wait(ctx); err != nil { + return + } + continue + } + switch state { + case 0: // ideal state + case 1: + log.ZWarn(ctx, "set seq cache lock not found", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + case 2: + log.ZWarn(ctx, "set seq cache lock to be held by someone else", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + default: + log.ZError(ctx, "set seq cache lock unknown state", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + } + return + } + log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) +} + +func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { + if size == 0 { + return 0 + } + var basicSize int64 + if msgprocessor.IsGroupConversationID(conversationID) { + basicSize = 200 + } else { + basicSize = 50 + } + basicSize += size + return basicSize +} + +func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errs.New("size must be greater than 0") + } + key := s.getSeqMallocKey(conversationID) + for i := 0; i < 10; i++ { + states, err := s.malloc(ctx, key, size) + if err != nil { + return 0, err + } + switch states[0] { + case 0: // success + return states[1], nil + case 1: // not found + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) + return seq, nil + case 2: // locked + if err := s.wait(ctx); err != nil { + return 0, err + } + continue + case 3: // exceeded cache max value + currSeq := states[1] + lastSeq := states[2] + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + if lastSeq == seq { + s.setSeqRetry(ctx, key, states[3], currSeq+size, seq+mallocSize) + return currSeq, nil + } else { + log.ZWarn(ctx, "malloc seq not equal cache last seq", nil, "conversationID", conversationID, "currSeq", currSeq, "lastSeq", lastSeq, "mallocSeq", seq) + s.setSeqRetry(ctx, key, states[3], seq+size, seq+mallocSize) + return seq, nil + } + default: + log.ZError(ctx, "malloc seq unknown state", nil, "state", states[0], "conversationID", conversationID, "size", size) + return 0, errs.New(fmt.Sprintf("unknown state: %d", states[0])) + } + } + log.ZError(ctx, "malloc seq retrying still failed", nil, "conversationID", conversationID, "size", size) + return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) +} + +func (s *SeqMalloc) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + return s.Malloc(ctx, conversationID, 0) +} diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go new file mode 100644 index 0000000000..1c11bad0b5 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_test.go @@ -0,0 +1,27 @@ +package redis + +import ( + "context" + "testing" + "time" +) + +func TestSeq(t *testing.T) { + ts := NewTestSeq() + for i := 1; i < 1000000; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "1", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second / 4) + } +} + +func TestDel(t *testing.T) { + ts := NewTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "1")) + +} diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq.go index 8592bf63cd..9b8c86ffe0 100644 --- a/pkg/common/storage/database/mgo/seq.go +++ b/pkg/common/storage/database/mgo/seq.go @@ -19,29 +19,24 @@ type SeqMongo struct { coll *mongo.Collection } -func (s *SeqMongo) MallocSeq(ctx context.Context, conversationID string, size int64) (int64, error) { - if size <= 0 { +func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { return 0, errors.New("size must be greater than 0") } + if size == 0 { + return s.GetMaxSeq(ctx, conversationID) + } filter := map[string]any{"conversation_id": conversationID} update := map[string]any{ "$inc": map[string]any{"max_seq": size}, "$set": map[string]any{"min_seq": 1}, } opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) - return mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) -} - -func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { - seq, err := s.MallocSeq(ctx, conversationID, size) + lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := seq - size + 1; i <= seq; i++ { - seqs = append(seqs, i) + return 0, err } - return seqs, nil + return lastSeq - size, nil } func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/seq_test.go similarity index 58% rename from pkg/common/storage/database/mgo/version_test.go rename to pkg/common/storage/database/mgo/seq_test.go index 236c61a2ce..abae2d1b15 100644 --- a/pkg/common/storage/database/mgo/version_test.go +++ b/pkg/common/storage/database/mgo/seq_test.go @@ -2,7 +2,6 @@ package mgo import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "testing" @@ -24,16 +23,17 @@ func Check(err error) { func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("version_test") - tmp, err := NewVersionLog(coll) + tmp, err := NewSeqMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - vl := tmp.(*VersionLogMgo) - res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) - if err != nil { - t.Log(err) - return + for i := 0; i < 10; i++ { + var size int64 = 100 + firstSeq, err := tmp.Malloc(context.Background(), "1", size) + if err != nil { + t.Log(err) + return + } + t.Logf("%d -> %d", firstSeq, firstSeq+size-1) } - t.Logf("%+v", res) } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index 869b607a5b..20fae3bb11 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -3,7 +3,7 @@ package database import "context" type Seq interface { - Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) GetMinSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeq(ctx context.Context, conversationID string, seq int64) error From a41c8c6a9d4cd7969dd9962866a930661834bf65 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 15:28:09 +0800 Subject: [PATCH 049/147] group version --- go.mod | 2 +- go.sum | 4 +-- internal/api/group.go | 8 +++--- internal/api/router.go | 4 +-- internal/rpc/friend/friend.go | 5 ++++ internal/rpc/group/notification.go | 28 ++++++++++--------- internal/rpc/group/sync.go | 8 +++--- .../storage/database/mgo/group_member.go | 16 +++++++---- .../storage/database/mgo/version_log.go | 4 +++ pkg/common/storage/database/version_log.go | 1 + 10 files changed, 49 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 4f4cd4b686..29e2fd43fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/protocol v0.0.69-alpha.12 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5a72e28f2c..a26e59d9ed 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= -github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.12 h1:3ZdwmD1y9vcduIC8o2EZS8Ds/fByqcuEFo+NkcBzgRo= +github.com/openimsdk/protocol v0.0.69-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/group.go b/internal/api/group.go index 0bf61c7878..91992004c5 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -187,10 +187,10 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { apiresp.GinSuccess(c, resp) } -func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) } -func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 69c026ffd6..d2fc9266fc 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,8 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) - groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) + groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 12107125c1..3d8970515d 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -51,6 +51,11 @@ type friendServer struct { webhookClient *webhook.Client } +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.Friend RedisConfig config.Redis diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 13e593c71a..4fae1ed2b4 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -289,10 +289,12 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } -func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, collName string, id string) { - for _, coll := range versionctx.GetVersionLog(ctx).Get() { +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { if coll.Name == collName && coll.Doc.DID == id { *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() return } } @@ -308,7 +310,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -392,7 +394,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, member.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -480,7 +482,7 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, req.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -494,7 +496,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } @@ -518,7 +520,7 @@ func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, } tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) } @@ -540,7 +542,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return } tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) } @@ -581,7 +583,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -606,7 +608,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -685,7 +687,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -709,7 +711,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -734,6 +736,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 4bd34eb4d4..d435530bd4 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -21,7 +21,7 @@ func (s *groupServer) idHash(ids []string) uint64 { return binary.BigEndian.Uint64(sum[:]) } -func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err @@ -34,7 +34,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req if req.IdHash == idHash { userIDs = nil } - return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + return &pbgroup.GetFullGroupMemberUserIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, @@ -42,7 +42,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req }, nil } -func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -55,7 +55,7 @@ func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgro if req.IdHash == idHash { groupIDs = nil } - return &pbgroup.GetIncrementalJoinGroupIDsResp{ + return &pbgroup.GetFullJoinGroupIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index a9ac262ab1..95c6e51a68 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -96,13 +96,19 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s } return mongoutil.IncrVersion(func() error { return mongoutil.DeleteMany(ctx, g.coll, filter) - }, func() error { - return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }, func() error { if len(userIDs) == 0 { - return nil + return g.member.Delete(ctx, groupID) + } else { + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + } + }, func() error { + for _, userID := range userIDs { + if err := g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } } - return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + return nil }) } @@ -112,7 +118,7 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us }, func() error { return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }, func() error { - return g.join.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + return g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index b53057fe4b..1d70f96f51 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -256,3 +256,7 @@ func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline ti }, }) } + +func (l *VersionLogMgo) Delete(ctx context.Context, dId string) error { + return mongoutil.DeleteOne(ctx, l.coll, bson.M{"d_id": dId}) +} diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index c9dc09540b..9d7bcc1724 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -15,4 +15,5 @@ type VersionLog interface { IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error + Delete(ctx context.Context, dId string) error } From bd4fb8a1a0e1ec5cbd32b611174d73b9e8db57ab Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 15:37:13 +0800 Subject: [PATCH 050/147] merge --- internal/rpc/friend/friend.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 619f8743b8..622e19f422 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -51,11 +51,6 @@ type friendServer struct { webhookClient *webhook.Client } -func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.Friend RedisConfig config.Redis From d55d416f6867fd62645895a771ceeb4222e8e3bc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 17:41:28 +0800 Subject: [PATCH 051/147] seq --- pkg/common/storage/cache/redis/seq.md | 36 ++++++++++++ pkg/common/storage/cache/redis/seq1.go | 9 +++ pkg/common/storage/cache/redis/seq_test.go | 64 +++++++++++++++++++--- pkg/common/storage/controller/msg.go | 1 - 4 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 pkg/common/storage/cache/redis/seq.md diff --git a/pkg/common/storage/cache/redis/seq.md b/pkg/common/storage/cache/redis/seq.md new file mode 100644 index 0000000000..9aa66a3265 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq.md @@ -0,0 +1,36 @@ + +### mongo +```go +type Seq struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} +``` + +```go +type Seq interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} +``` + +1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1 +2. GetMaxSeq 获取申请的seq的最大值，在发消息的seq小于这个值 +3. GetMinSeq 获取最小的seq，用于拉取历史消息 +4. SetMinSeq 设置最小的seq，用于拉取历史消息 + +### redis +```go +type RedisSeq struct { + Curr int64 // 当前的最大seq + Last int64 // mongodb中申请的最大seq + Lock *int64 // 锁，用于在mongodb中申请seq +} +``` + +1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1，如果redis中申请数量够用，直接返回，并自增对应数量。如果redis中申请数量不够用，加锁，从mongodb中申请seq。 +2. GetMaxSeq 获取已发消息的最大seq就是Curr的值。如果redis中缓存不存在就通过mongodb获取最大seq。存储在redis中。其中Curr和Last都是这个seq值。 +3. GetMinSeq, SetMinSeq用之前rockscache的方案。 \ No newline at end of file diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq1.go index 6374f53ccf..316dea487b 100644 --- a/pkg/common/storage/cache/redis/seq1.go +++ b/pkg/common/storage/cache/redis/seq1.go @@ -47,6 +47,12 @@ func NewTestSeq() *SeqMalloc { } } +type RedisSeq struct { + Curr int64 + Last int64 + Lock *int64 +} + type SeqMalloc struct { rdb redis.UniversalClient mgo database.Seq @@ -118,6 +124,7 @@ end local curr_seq = tonumber(redis.call("HGET", key, "CURR")) local last_seq = tonumber(redis.call("HGET", key, "LAST")) if size == 0 then + redis.call("EXPIRE", key, dataSecond) table.insert(result, 0) table.insert(result, curr_seq) table.insert(result, last_seq) @@ -136,6 +143,7 @@ if max_seq > last_seq then return result end redis.call("HSET", key, "CURR", max_seq) +redis.call("EXPIRE", key, dataSecond) table.insert(result, 0) table.insert(result, curr_seq) table.insert(result, last_seq) @@ -219,6 +227,7 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) return seq, nil case 2: // locked + fmt.Println("locked----->", "conversationID", conversationID, "size", size) if err := s.wait(ctx); err != nil { return 0, err } diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go index 1c11bad0b5..6fff6019a3 100644 --- a/pkg/common/storage/cache/redis/seq_test.go +++ b/pkg/common/storage/cache/redis/seq_test.go @@ -2,22 +2,72 @@ package redis import ( "context" + "strconv" + "sync" + "sync/atomic" "testing" "time" ) func TestSeq(t *testing.T) { ts := NewTestSeq() - for i := 1; i < 1000000; i++ { - var size int64 = 100 - first, err := ts.Malloc(context.Background(), "1", size) - if err != nil { - t.Logf("[%d] %s", i, err) + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 256 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 1 + cID := strconv.Itoa(index * 100) + for i := 1; ; i++ { + first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + //first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) } - t.Logf("[%d] %d -> %d", i, first+1, first+size) - time.Sleep(time.Second / 4) } + + //for i := 1; i < 1000000; i++ { + // var size int64 = 100 + // first, err := ts.Malloc(context.Background(), "1", size) + // if err != nil { + // t.Logf("[%d] %s", i, err) + // return + // } + // t.Logf("[%d] %d -> %d", i, first+1, first+size) + // time.Sleep(time.Second / 4) + //} } func TestDel(t *testing.T) { diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index f0efe5efdf..c5c5462aab 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -349,7 +349,6 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - // TODO set SEQ currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) if err != nil && errs.Unwrap(err) != redis.Nil { From a2a28b43c508225bbd6fb1b593c1f6584f491a9a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 18 Jun 2024 16:46:06 +0800 Subject: [PATCH 052/147] seq --- internal/msgtransfer/init.go | 7 +- internal/rpc/msg/server.go | 7 +- pkg/common/storage/cache/cachekey/seq.go | 16 +- pkg/common/storage/cache/cachekey/seq1.go | 5 - pkg/common/storage/cache/redis/seq.md | 36 --- pkg/common/storage/cache/redis/seq1.go.txt | 246 ------------------ .../redis/{seq1.go => seq_conversation.go} | 109 ++++---- .../cache/redis/seq_conversation_test.go | 109 ++++++++ pkg/common/storage/cache/redis/seq_test.go | 77 ------ pkg/common/storage/cache/seq.go | 14 +- pkg/common/storage/cache/seq_conversation.go | 10 + pkg/common/storage/controller/msg.go | 126 +++------ .../mgo/{seq.go => seq_conversation.go} | 43 ++- .../{seq_test.go => seq_conversation_test.go} | 0 pkg/common/storage/database/name.go | 2 +- pkg/common/storage/database/seq.go | 2 +- pkg/common/storage/model/seq.go | 2 +- tools/seq/internal/main.go | 152 +++++++++++ tools/seq/main.go | 16 ++ 19 files changed, 443 insertions(+), 536 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/seq.md delete mode 100644 pkg/common/storage/cache/redis/seq1.go.txt rename pkg/common/storage/cache/redis/{seq1.go => seq_conversation.go} (68%) create mode 100644 pkg/common/storage/cache/redis/seq_conversation_test.go delete mode 100644 pkg/common/storage/cache/redis/seq_test.go create mode 100644 pkg/common/storage/cache/seq_conversation.go rename pkg/common/storage/database/mgo/{seq.go => seq_conversation.go} (50%) rename pkg/common/storage/database/mgo/{seq_test.go => seq_conversation_test.go} (100%) create mode 100644 tools/seq/internal/main.go create mode 100644 tools/seq/main.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 65d04f3810..e90c2288c8 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -87,7 +87,12 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index f1fb28ffff..7cffb23a9c 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -91,7 +91,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index 3f0ce98a4d..ded0286d84 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -15,24 +15,24 @@ package cachekey const ( - maxSeq = "MAX_SEQ:" - minSeq = "MIN_SEQ:" - conversationUserMinSeq = "CON_USER_MIN_SEQ:" - hasReadSeq = "HAS_READ_SEQ:" + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" ) func GetMaxSeqKey(conversationID string) string { - return maxSeq + conversationID + return MaxSeq + conversationID } func GetMinSeqKey(conversationID string) string { - return minSeq + conversationID + return MinSeq + conversationID } func GetHasReadSeqKey(conversationID string, userID string) string { - return hasReadSeq + userID + ":" + conversationID + return HasReadSeq + userID + ":" + conversationID } func GetConversationUserMinSeqKey(conversationID, userID string) string { - return conversationUserMinSeq + conversationID + "u:" + userID + return ConversationUserMinSeq + conversationID + "u:" + userID } diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go index df3086d326..04274db55a 100644 --- a/pkg/common/storage/cache/cachekey/seq1.go +++ b/pkg/common/storage/cache/cachekey/seq1.go @@ -2,7 +2,6 @@ package cachekey const ( MallocSeq = "MALLOC_SEQ:" - MallocSeqLock = "MALLOC_SEQ_LOCK:" MallocMinSeqLock = "MALLOC_MIN_SEQ:" ) @@ -10,10 +9,6 @@ func GetMallocSeqKey(conversationID string) string { return MallocSeq + conversationID } -func GetMallocSeqLockKey(conversationID string) string { - return MallocSeqLock + conversationID -} - func GetMallocMinSeqKey(conversationID string) string { return MallocMinSeqLock + conversationID } diff --git a/pkg/common/storage/cache/redis/seq.md b/pkg/common/storage/cache/redis/seq.md deleted file mode 100644 index 9aa66a3265..0000000000 --- a/pkg/common/storage/cache/redis/seq.md +++ /dev/null @@ -1,36 +0,0 @@ - -### mongo -```go -type Seq struct { - ConversationID string `bson:"conversation_id"` - MaxSeq int64 `bson:"max_seq"` - MinSeq int64 `bson:"min_seq"` -} -``` - -```go -type Seq interface { - Malloc(ctx context.Context, conversationID string, size int64) (int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, seq int64) error -} -``` - -1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1 -2. GetMaxSeq 获取申请的seq的最大值，在发消息的seq小于这个值 -3. GetMinSeq 获取最小的seq，用于拉取历史消息 -4. SetMinSeq 设置最小的seq，用于拉取历史消息 - -### redis -```go -type RedisSeq struct { - Curr int64 // 当前的最大seq - Last int64 // mongodb中申请的最大seq - Lock *int64 // 锁，用于在mongodb中申请seq -} -``` - -1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1，如果redis中申请数量够用，直接返回，并自增对应数量。如果redis中申请数量不够用，加锁，从mongodb中申请seq。 -2. GetMaxSeq 获取已发消息的最大seq就是Curr的值。如果redis中缓存不存在就通过mongodb获取最大seq。存储在redis中。其中Curr和Last都是这个seq值。 -3. GetMinSeq, SetMinSeq用之前rockscache的方案。 \ No newline at end of file diff --git a/pkg/common/storage/cache/redis/seq1.go.txt b/pkg/common/storage/cache/redis/seq1.go.txt deleted file mode 100644 index 9ccb83e668..0000000000 --- a/pkg/common/storage/cache/redis/seq1.go.txt +++ /dev/null @@ -1,246 +0,0 @@ -package redis - -import ( - "context" - "errors" - "fmt" - "github.com/dtm-labs/rockscache" - "github.com/google/uuid" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/tools/errs" - "github.com/redis/go-redis/v9" - "strconv" - "time" -) - -var errLock = errors.New("lock failed") - -type MallocSeq interface { - Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, seq int64) error -} - -func NewSeqCache1(rdb redis.UniversalClient, mgo database.Seq) MallocSeq { - opt := rockscache.NewDefaultOptions() - opt.EmptyExpire = time.Second * 3 - opt.Delay = time.Second / 2 - return &seqCache1{ - rdb: rdb, - mgo: mgo, - rocks: rockscache.NewClient(rdb, opt), - lockExpire: time.Minute * 1, - seqExpire: time.Hour * 24 * 7, - minSeqExpire: time.Hour * 1, - groupMinNum: 1000, - userMinNum: 100, - } -} - -type seqCache1 struct { - rdb redis.UniversalClient - rocks *rockscache.Client - mgo database.Seq - lockExpire time.Duration - seqExpire time.Duration - minSeqExpire time.Duration - groupMinNum int64 - userMinNum int64 -} - -/* -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -*/ - -func (s *seqCache1) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - for i := 0; i < 10; i++ { - res, err := s.rdb.LIndex(ctx, cachekey.GetMallocSeqKey(conversationID), 0).Int64() - if err == nil { - return res, nil - } else if !errors.Is(err, redis.Nil) { - return 0, errs.Wrap(err) - } - - if err := s.mallocSeq(ctx, conversationID, 0, nil); err != nil { - return 0, err - } - } - return 0, errs.New("get max seq timeout") -} - -func (s *seqCache1) unlock(ctx context.Context, key string, owner string) error { - script := ` -local value = redis.call("GET", KEYS[1]) -if value == false then - return 0 -end -if value == ARGV[1] then - redis.call("DEL", KEYS[1]) - return 1 -end -return 2 -` - state, err := s.rdb.Eval(ctx, script, []string{key}, owner).Int() - if err != nil { - return errs.Wrap(err) - } - switch state { - case 0: - return errs.Wrap(redis.Nil) - case 1: - return nil - case 2: - return errs.New("not the lock holder") - default: - return errs.New(fmt.Sprintf("unknown state: %d", state)) - } -} - -func (s *seqCache1) initMallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { - owner := uuid.New().String() - ok, err := s.rdb.SetNX(ctx, cachekey.GetMallocSeqLockKey(conversationID), owner, s.lockExpire).Result() - if err != nil { - return nil, err - } - - seq, err := s.mgo.Malloc(ctx, conversationID, size) - if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := seq - size + 1; i <= seq; i++ { - seqs = append(seqs, i) - } - return seqs, nil -} - -func (s *seqCache1) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return getCache[int64](ctx, s.rocks, cachekey.GetMallocMinSeqKey(conversationID), s.minSeqExpire, func(ctx context.Context) (int64, error) { - return s.mgo.GetMinSeq(ctx, conversationID) - }) -} - -func (s *seqCache1) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { - if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { - return err - } - return s.deleteMinSeqCache(ctx, conversationID) -} - -func (s *seqCache1) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { - if size <= 0 { - return nil, errs.Wrap(errors.New("size must be greater than 0")) - } - seqKey := cachekey.GetMallocSeqKey(conversationID) - lockKey := cachekey.GetMallocSeqLockKey(conversationID) - for i := 0; i < 10; i++ { - seqs, err := s.lpop(ctx, seqKey, lockKey, size) - if err != nil { - return nil, err - } - if len(seqs) < int(size) { - if err := s.mallocSeq(ctx, conversationID, size, &seqs); err != nil { - return nil, err - } - } - if len(seqs) >= int(size) { - return seqs, nil - } - } - return nil, errs.ErrInternalServer.WrapMsg("malloc seq failed") -} - -func (s *seqCache1) push(ctx context.Context, seqKey string, seqs []int64) error { - script := ` -redis.call("DEL", KEYS[1]) -for i = 2, #ARGV do - redis.call("RPUSH", KEYS[1], ARGV[i]) -end -redis.call("EXPIRE", KEYS[1], ARGV[1]) -return 1 -` - argv := make([]any, 0, 1+len(seqs)) - argv = append(argv, s.seqExpire.Seconds()) - for _, seq := range seqs { - argv = append(argv, seq) - } - err := s.rdb.Eval(ctx, script, []string{seqKey}, argv...).Err() - return errs.Wrap(err) -} - -func (s *seqCache1) lpop(ctx context.Context, seqKey, lockKey string, size int64) ([]int64, error) { - script := ` -local result = redis.call("LRANGE", KEYS[1], 0, ARGV[1]-1) -if #result == 0 then - return result -end -redis.call("LTRIM", KEYS[1], #result, -1) -if redis.call("LLEN", KEYS[1]) == 0 then - redis.call("DEL", KEYS[2]) -end -return result -` - res, err := s.rdb.Eval(ctx, script, []string{seqKey, lockKey}, size).Int64Slice() - if err != nil { - return nil, errs.Wrap(err) - } - return res, nil -} - -func (s *seqCache1) getMongoStepSize(conversationID string, size int64) int64 { - var num int64 - if msgprocessor.IsGroupConversationID(conversationID) { - num = s.groupMinNum - } else { - num = s.userMinNum - } - if size > num { - num += size - } - return num -} - -func (s *seqCache1) mallocSeq(ctx context.Context, conversationID string, size int64, seqs *[]int64) error { - var delMinSeqKey bool - _, err := getCache[string](ctx, s.rocks, cachekey.GetMallocSeqLockKey(conversationID), s.lockExpire, func(ctx context.Context) (string, error) { - res, err := s.mgo.Malloc(ctx, conversationID, s.getMongoStepSize(conversationID, size)) - if err != nil { - return "", err - } - delMinSeqKey = res[0] == 1 - if seqs != nil && size > 0 { - if len(*seqs) > 0 && (*seqs)[len(*seqs)-1]+1 == res[0] { - n := size - int64(len(*seqs)) - *seqs = append(*seqs, res[:n]...) - res = res[n:] - } else { - *seqs = res[:size] - res = res[size:] - } - } - if err := s.push(ctx, cachekey.GetMallocSeqKey(conversationID), res); err != nil { - return "", err - } - return strconv.Itoa(int(time.Now().UnixMicro())), nil - }) - if delMinSeqKey { - s.deleteMinSeqCache(ctx, conversationID) - } - return err -} - -func (s *seqCache1) deleteMinSeqCache(ctx context.Context, conversationID string) error { - return s.rocks.TagAsDeleted2(ctx, cachekey.GetMallocMinSeqKey(conversationID)) -} diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq_conversation.go similarity index 68% rename from pkg/common/storage/cache/redis/seq1.go rename to pkg/common/storage/cache/redis/seq_conversation.go index 316dea487b..034462fd1d 100644 --- a/pkg/common/storage/cache/redis/seq1.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -3,74 +3,68 @@ package redis import ( "context" "fmt" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "time" ) -type RedisHash struct { - NextSeq int64 - LastSeq int64 +func NewSeqConversationCacheRedis(rdb redis.UniversalClient, mgo database.SeqConversation) cache.SeqConversationCache { + return &seqConversationCacheRedis{ + rdb: rdb, + mgo: mgo, + lockTime: time.Second * 3, + dataTime: time.Hour * 24 * 365, + minSeqExpireTime: time.Hour, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } } -func NewTestSeq() *SeqMalloc { - mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) - if err != nil { - panic(err) - } - model, err := mgo.NewSeqMongo(mgocli.Database("openim_v3")) - if err != nil { - panic(err) - } - opt := &redis.Options{ - Addr: "172.16.8.48:16379", - Password: "openIM123", - DB: 1, - } - rdb := redis.NewClient(opt) - if err := rdb.Ping(context.Background()).Err(); err != nil { - panic(err) - } - return &SeqMalloc{ - rdb: rdb, - mgo: model, - //lockTime: time.Second * 30, - lockTime: time.Second * 60 * 60 * 24 * 1, - dataTime: time.Second * 60 * 60 * 24 * 7, - } +type seqConversationCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqConversation + rocks *rockscache.Client + lockTime time.Duration + dataTime time.Duration + minSeqExpireTime time.Duration +} + +func (s *seqConversationCacheRedis) getMinSeqKey(conversationID string) string { + return cachekey.GetMallocMinSeqKey(conversationID) } -type RedisSeq struct { - Curr int64 - Last int64 - Lock *int64 +func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + if err := s.rocks.TagAsDeleted2(ctx, s.getMinSeqKey(conversationID)); err != nil { + return errs.Wrap(err) + } + return nil } -type SeqMalloc struct { - rdb redis.UniversalClient - mgo database.Seq - lockTime time.Duration - dataTime time.Duration +func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache(ctx, s.rocks, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) } -func (s *SeqMalloc) getSeqMallocKey(conversationID string) string { +func (s *seqConversationCacheRedis) getSeqMallocKey(conversationID string) string { return cachekey.GetMallocSeqKey(conversationID) } -func (s *SeqMalloc) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { +func (s *seqConversationCacheRedis) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { if lastSeq < currSeq { return 0, errs.New("lastSeq must be greater than currSeq") } - // 0： 成功 - // 1： 成功 锁过期，但未被其他人锁 - // 2： 已经被锁，但是锁的不是自己 + // 0: success + // 1: success the lock has expired, but has not been locked by anyone else + // 2: already locked, but not by yourself script := ` local key = KEYS[1] local lockValue = ARGV[1] @@ -97,12 +91,12 @@ return 0 return result, nil } -// malloc size=0为获取当前seq size>0为分配seq -func (s *SeqMalloc) malloc(ctx context.Context, key string, size int64) ([]int64, error) { - // 0： 成功 - // 1： 需要获取，并加锁 - // 2： 已经被锁 - // 3： 超过最大值，并加锁 +// malloc size=0 is to get the current seq size>0 is to allocate seq +func (s *seqConversationCacheRedis) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0: success + // 1: need to obtain and lock + // 2: already locked + // 3: exceeded the maximum value and locked script := ` local key = KEYS[1] local size = tonumber(ARGV[1]) @@ -156,7 +150,7 @@ return result return result, nil } -func (s *SeqMalloc) wait(ctx context.Context) error { +func (s *seqConversationCacheRedis) wait(ctx context.Context) error { timer := time.NewTimer(time.Second / 4) defer timer.Stop() select { @@ -167,7 +161,7 @@ func (s *SeqMalloc) wait(ctx context.Context) error { } } -func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { +func (s *seqConversationCacheRedis) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { for i := 0; i < 10; i++ { state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) if err != nil { @@ -191,13 +185,13 @@ func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, cu log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) } -func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { +func (s *seqConversationCacheRedis) getMallocSize(conversationID string, size int64) int64 { if size == 0 { return 0 } var basicSize int64 if msgprocessor.IsGroupConversationID(conversationID) { - basicSize = 200 + basicSize = 100 } else { basicSize = 50 } @@ -205,7 +199,7 @@ func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { return basicSize } -func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { +func (s *seqConversationCacheRedis) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errs.New("size must be greater than 0") } @@ -227,7 +221,6 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) return seq, nil case 2: // locked - fmt.Println("locked----->", "conversationID", conversationID, "size", size) if err := s.wait(ctx); err != nil { return 0, err } @@ -257,6 +250,6 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) } -func (s *SeqMalloc) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationCacheRedis) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { return s.Malloc(ctx, conversationID, 0) } diff --git a/pkg/common/storage/cache/redis/seq_conversation_test.go b/pkg/common/storage/cache/redis/seq_conversation_test.go new file mode 100644 index 0000000000..1a40624b8c --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation_test.go @@ -0,0 +1,109 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +func newTestSeq() *seqConversationCacheRedis { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqConversationMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return NewSeqConversationCacheRedis(rdb, model).(*seqConversationCacheRedis) +} + +func TestSeq(t *testing.T) { + ts := newTestSeq() + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 128 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 10 + cID := strconv.Itoa(index * 1) + for i := 1; ; i++ { + //first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) + } + } +} + +func TestDel(t *testing.T) { + ts := newTestSeq() + for i := 1; i < 100; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "100", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second) + } +} + +func TestSeqMalloc(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "100")) +} + +func TestMinSeq(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMinSeq(context.Background(), "10000000")) +} diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go deleted file mode 100644 index 6fff6019a3..0000000000 --- a/pkg/common/storage/cache/redis/seq_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package redis - -import ( - "context" - "strconv" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestSeq(t *testing.T) { - ts := NewTestSeq() - var ( - wg sync.WaitGroup - speed atomic.Int64 - ) - - const count = 256 - wg.Add(count) - for i := 0; i < count; i++ { - index := i + 1 - go func() { - defer wg.Done() - var size int64 = 1 - cID := strconv.Itoa(index * 100) - for i := 1; ; i++ { - first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo - //first, err := ts.Malloc(context.Background(), cID, size) // redis - if err != nil { - t.Logf("[%d-%d] %s %s", index, i, cID, err) - return - } - speed.Add(size) - _ = first - //t.Logf("[%d] %d -> %d", i, first+1, first+size) - } - }() - } - - done := make(chan struct{}) - - go func() { - wg.Wait() - close(done) - }() - - ticker := time.NewTicker(time.Second) - - for { - select { - case <-done: - ticker.Stop() - return - case <-ticker.C: - value := speed.Swap(0) - t.Logf("speed: %d/s", value) - } - } - - //for i := 1; i < 1000000; i++ { - // var size int64 = 100 - // first, err := ts.Malloc(context.Background(), "1", size) - // if err != nil { - // t.Logf("[%d] %s", i, err) - // return - // } - // t.Logf("[%d] %d -> %d", i, first+1, first+size) - // time.Sleep(time.Second / 4) - //} -} - -func TestDel(t *testing.T) { - ts := NewTestSeq() - t.Log(ts.GetMaxSeq(context.Background(), "1")) - -} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go index 091b318c89..46e33c9357 100644 --- a/pkg/common/storage/cache/seq.go +++ b/pkg/common/storage/cache/seq.go @@ -5,13 +5,13 @@ import ( ) type SeqCache interface { - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error + //SetMinSeqs(ctx context.Context, seqs map[string]int64) error + //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMinSeq(ctx context.Context, conversationID string) (int64, error) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error diff --git a/pkg/common/storage/cache/seq_conversation.go b/pkg/common/storage/cache/seq_conversation.go new file mode 100644 index 0000000000..5d38537a9e --- /dev/null +++ b/pkg/common/storage/cache/seq_conversation.go @@ -0,0 +1,10 @@ +package cache + +import "context" + +type SeqConversationCache interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index c5c5462aab..3a29f6786c 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -108,7 +108,7 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -129,28 +129,19 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach msgDocDatabase: msgDocModel, msg: msg, seq: seq, + seqConversation: seqConversation, producer: producerToRedis, producerToMongo: producerToMongo, producerToPush: producerToPush, }, nil } -//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) { -// msgDocModel, err := database.NewMsgMongo(database) -// if err != nil { -// return nil, err -// } -// //todo MsgCacheTimeout -// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline) -// seq := cache.NewSeqCache(rdb) -// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig) -//} - type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache seq cache.SeqCache + seqConversation cache.SeqConversationCache producer *kafka.Producer producerToMongo *kafka.Producer producerToPush *kafka.Producer @@ -349,10 +340,9 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - // TODO set SEQ - currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { - log.ZError(ctx, "storage.seq.GetMaxSeq", err) + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) return 0, false, err } lenList := len(msgs) @@ -362,9 +352,6 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa if lenList < 1 { return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() } - if errs.Unwrap(err) == redis.Nil { - isNew = true - } lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) for _, m := range msgs { @@ -380,13 +367,6 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - - err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq) - if err != nil { - log.ZError(ctx, "storage.seq.SetMaxSeq error", err, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) if err != nil { log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) @@ -519,8 +499,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq > minSeq { @@ -531,8 +511,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end) return 0, 0, nil, nil } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq) @@ -572,11 +552,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin var successMsgs []*sdkws.MsgData log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil { - if err != redis.Nil { - - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } + if err != nil && !errors.Is(err, redis.Nil) { + log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) } successMsgs = append(successMsgs, cachedMsgs...) log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) @@ -600,12 +577,12 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq < minSeq { @@ -649,7 +626,7 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont if minSeq == 0 { return nil } - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) { @@ -803,7 +780,7 @@ func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationI func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) { for _, conversationID := range conversationIDs { - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { if err == redis.Nil { log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID) @@ -812,7 +789,7 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u } continue } - if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { + if err := db.seqConversation.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1) } } @@ -823,29 +800,37 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u //} func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMaxSeqs(ctx, conversationIDs) + result := make(map[string]int64) + for _, conversationID := range conversationIDs { + if result[conversationID] != 0 { + continue + } + seq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { + return nil, err + } + result[conversationID] = seq + } + return result, nil } func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMaxSeq(ctx, conversationID) + return db.seqConversation.GetMaxSeq(ctx, conversationID) } func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return db.seq.SetMinSeqs(ctx, seqs) + for conversationID, seq := range seqs { + if err := db.seqConversation.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return nil } -//func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { -// return db.seq.GetMinSeqs(ctx, conversationIDs) -//} -// -//func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { -// return db.seq.GetMinSeq(ctx, conversationID) -//} - func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) } @@ -895,11 +880,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context if err != nil { return } - minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID) + minSeqCache, err = db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { return } - maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID) + maxSeqCache, err = db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { return } @@ -1011,33 +996,8 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d } } -//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) { -// var ( -// docNum int -// msgNum int -// start = time.Now() -// ) -// for { -// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return err -// } -// if len(msgs) == 0 { -// return nil -// } -// for _, msg := range msgs { -// num, err := db.deleteOneMsg(ctx, ts, msg) -// if err != nil { -// return err -// } -// docNum++ -// msgNum += num -// } -// } -//} - func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { - dbSeq, err := db.seq.GetMinSeq(ctx, conversationID) + dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { return nil @@ -1047,5 +1007,5 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin if dbSeq >= seq { return nil } - return db.seq.SetMinSeq(ctx, conversationID, seq) + return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq_conversation.go similarity index 50% rename from pkg/common/storage/database/mgo/seq.go rename to pkg/common/storage/database/mgo/seq_conversation.go index 9b8c86ffe0..c9a3ac41a9 100644 --- a/pkg/common/storage/database/mgo/seq.go +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -3,6 +3,7 @@ package mgo import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" @@ -10,16 +11,24 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -func NewSeqMongo(db *mongo.Database) (*SeqMongo, error) { - coll := db.Collection("seq") - return &SeqMongo{coll: coll}, nil +func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqConversationMongo{coll: coll}, nil } -type SeqMongo struct { +type seqConversationMongo struct { coll *mongo.Collection } -func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { +func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errors.New("size must be greater than 0") } @@ -29,7 +38,7 @@ func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64 filter := map[string]any{"conversation_id": conversationID} update := map[string]any{ "$inc": map[string]any{"max_seq": size}, - "$set": map[string]any{"min_seq": 1}, + "$set": map[string]any{"min_seq": int64(0)}, } opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) @@ -39,7 +48,19 @@ func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64 return lastSeq - size, nil } -func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationMongo) MallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { + first, err := s.Malloc(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := int64(0); i < size; i++ { + seqs = append(seqs, first+i+1) + } + return seqs, nil +} + +func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) if err == nil { return seq, nil @@ -50,7 +71,7 @@ func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, } } -func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) if err == nil { return seq, nil @@ -61,10 +82,10 @@ func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, } } -func (s *SeqMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { +func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) } -func (s *SeqMongo) GetConversation(ctx context.Context, conversationID string) (*model.Seq, error) { - return mongoutil.FindOne[*model.Seq](ctx, s.coll, bson.M{"conversation_id": conversationID}) +func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { + return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID}) } diff --git a/pkg/common/storage/database/mgo/seq_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go similarity index 100% rename from pkg/common/storage/database/mgo/seq_test.go rename to pkg/common/storage/database/mgo/seq_conversation_test.go diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index f496d4b41a..68fa5af02c 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -14,5 +14,5 @@ const ( LogName = "log" ObjectName = "s3" UserName = "user" - SeqName = "seq" + SeqConversationName = "seq" ) diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index 20fae3bb11..fdbc2f8f33 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -2,7 +2,7 @@ package database import "context" -type Seq interface { +type SeqConversation interface { Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) GetMinSeq(ctx context.Context, conversationID string) (int64, error) diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go index 3db0091676..1dc75eff18 100644 --- a/pkg/common/storage/model/seq.go +++ b/pkg/common/storage/model/seq.go @@ -1,6 +1,6 @@ package model -type Seq struct { +type SeqConversation struct { ConversationID string `bson:"conversation_id"` MaxSeq int64 `bson:"max_seq"` MinSeq int64 `bson:"min_seq"` diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go new file mode 100644 index 0000000000..5cfd9fd45d --- /dev/null +++ b/tools/seq/internal/main.go @@ -0,0 +1,152 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "gopkg.in/yaml.v3" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +const batchSize = 5 + +func readConfig[T any](dir string, name string) (*T, error) { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return nil, err + } + var conf T + if err := yaml.Unmarshal(data, &conf); err != nil { + return nil, err + } + return &conf, nil +} + +func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { + var ( + cursor uint64 + keys []string + err error + ) + ctx := context.Background() + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { + return err + } + delKey := make(map[string]struct{}) + if len(keys) > 0 { + for _, key := range keys { + if err := fn(ctx, key, delKey); err != nil { + return err + } + } + } + if len(delKey) > 0 { + //if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { + // return err + //} + } + if cursor == 0 { + return nil + } + } +} + +func Main(conf string) error { + redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) + if err != nil { + return err + } + mongodbConfig, err := readConfig[config.Mongo](conf, cmd.MongodbConfigFileName) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, redisConfig.Build()) + if err != nil { + return err + } + mgocli, err := mongoutil.NewMongoDB(ctx, mongodbConfig.Build()) + if err != nil { + return err + } + if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { + return err + } + coll := mgocli.GetDB().Collection(database.SeqConversationName) + const prefix = cachekey.MaxSeq + return redisKey(rdb, prefix, func(ctx context.Context, key string, delKey map[string]struct{}) error { + conversationId := strings.TrimPrefix(key, prefix) + delKey[key] = struct{}{} + maxValue, err := rdb.Get(ctx, key).Result() + if err != nil { + return err + } + seq, err := strconv.Atoi(maxValue) + if err != nil { + return fmt.Errorf("invalid max seq %s", maxValue) + } + if seq == 0 { + return nil + } + if seq < 0 { + return fmt.Errorf("invalid max seq %s", maxValue) + } + var ( + minSeq int64 + maxSeq = int64(seq) + ) + minKey := cachekey.MinSeq + conversationId + delKey[minKey] = struct{}{} + minValue, err := rdb.Get(ctx, minKey).Result() + if err == nil { + seq, err := strconv.Atoi(minValue) + if err != nil { + return fmt.Errorf("invalid min seq %s", minValue) + } + if seq < 0 { + return fmt.Errorf("invalid min seq %s", minValue) + } + minSeq = int64(seq) + } else if !errors.Is(err, redis.Nil) { + return err + } + if maxSeq < minSeq { + return fmt.Errorf("invalid max seq %d < min seq %d", maxSeq, minSeq) + } + res, err := mongoutil.FindOne[*model.SeqConversation](ctx, coll, bson.M{"conversation_id": conversationId}, nil) + if err == nil { + if res.MaxSeq < int64(seq) { + _, err = coll.UpdateOne(ctx, bson.M{"conversation_id": conversationId}, bson.M{"$set": bson.M{"max_seq": maxSeq, "min_seq": minSeq}}) + } + return err + } else if errors.Is(err, mongo.ErrNoDocuments) { + res = &model.SeqConversation{ + ConversationID: conversationId, + MaxSeq: maxSeq, + MinSeq: minSeq, + } + _, err := coll.InsertOne(ctx, res) + return err + } else { + return err + } + }) +} diff --git a/tools/seq/main.go b/tools/seq/main.go new file mode 100644 index 0000000000..ca5a043e77 --- /dev/null +++ b/tools/seq/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" +) + +func main() { + var config string + flag.StringVar(&config, "redis", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + flag.Parse() + if err := internal.Main(config); err != nil { + fmt.Println("seq task", err) + } +} From cce382d0c37cd954fa38af1936813d8b7f056096 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 18 Jun 2024 18:06:40 +0800 Subject: [PATCH 053/147] seq --- .../database/mgo/seq_conversation_test.go | 8 +- start-config.yml | 1 + tools/seq/internal/main.go | 80 +++++++++++++++++-- tools/seq/main.go | 11 ++- 4 files changed, 83 insertions(+), 17 deletions(-) diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index abae2d1b15..e6466a1c60 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -15,15 +15,9 @@ func Result[V any](val V, err error) V { return val } -func Check(err error) { - if err != nil { - panic(err) - } -} - func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - tmp, err := NewSeqMongo(cli.Database("openim_v3")) + tmp, err := NewSeqConversationMongo(cli.Database("openim_v3")) if err != nil { panic(err) } diff --git a/start-config.yml b/start-config.yml index a9c412b33f..21436d7a9a 100644 --- a/start-config.yml +++ b/start-config.yml @@ -14,4 +14,5 @@ serviceBinaries: toolBinaries: - check-free-memory - check-component + - seq maxFileDescriptors: 10000 diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 5cfd9fd45d..54a7d477da 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -12,9 +12,11 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "gopkg.in/yaml.v3" "os" "path/filepath" @@ -23,7 +25,12 @@ import ( "time" ) -const batchSize = 5 +const ( + batchSize = 100 + dataVersionCollection = "data_version" + seqKey = "seq" + seqVersion = 38 +) func readConfig[T any](dir string, name string) (*T, error) { data, err := os.ReadFile(filepath.Join(dir, name)) @@ -37,7 +44,7 @@ func readConfig[T any](dir string, name string) (*T, error) { return &conf, nil } -func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { +func redisKey(rdb redis.UniversalClient, prefix string, del time.Duration, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { var ( cursor uint64 keys []string @@ -58,9 +65,20 @@ func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Cont } } if len(delKey) > 0 { - //if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { - // return err - //} + delKeys := datautil.Keys(delKey) + if del < time.Second { + if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { + return err + } + } else { + pipe := rdb.Pipeline() + for _, key := range delKeys { + pipe.Expire(ctx, key, del) + } + if _, err := pipe.Exec(ctx); err != nil { + return err + } + } } if cursor == 0 { return nil @@ -68,7 +86,7 @@ func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Cont } } -func Main(conf string) error { +func Main(conf string, del time.Duration) error { redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) if err != nil { return err @@ -87,12 +105,22 @@ func Main(conf string) error { if err != nil { return err } + versionColl := mgocli.GetDB().Collection(dataVersionCollection) + converted, err := CheckVersion(versionColl, seqKey, seqVersion) + if err != nil { + return err + } + if converted { + fmt.Println("[seq] seq data has been converted") + return nil + } if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { return err } coll := mgocli.GetDB().Collection(database.SeqConversationName) const prefix = cachekey.MaxSeq - return redisKey(rdb, prefix, func(ctx context.Context, key string, delKey map[string]struct{}) error { + fmt.Println("start to convert seq conversation") + err = redisKey(rdb, prefix, del, func(ctx context.Context, key string, delKey map[string]struct{}) error { conversationId := strings.TrimPrefix(key, prefix) delKey[key] = struct{}{} maxValue, err := rdb.Get(ctx, key).Result() @@ -149,4 +177,42 @@ func Main(conf string) error { return err } }) + if err != nil { + return err + } + fmt.Println("convert seq conversation success") + return SetVersion(versionColl, seqKey, seqVersion) +} + +func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { + type VersionTable struct { + Key string `bson:"key"` + Value string `bson:"value"` + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + res, err := mongoutil.FindOne[VersionTable](ctx, coll, bson.M{"key": key}) + if err == nil { + ver, err := strconv.Atoi(res.Value) + if err != nil { + return false, fmt.Errorf("version %s parse error %w", res.Value, err) + } + if ver >= currentVersion { + return true, nil + } + return false, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return false, nil + } else { + return false, err + } +} + +func SetVersion(coll *mongo.Collection, key string, version int) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + option := options.Update().SetUpsert(true) + filter := bson.M{"key": key, "value": strconv.Itoa(version)} + update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}} + return mongoutil.UpdateOne(ctx, coll, filter, update, false, option) } diff --git a/tools/seq/main.go b/tools/seq/main.go index ca5a043e77..6bb4b76578 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -4,13 +4,18 @@ import ( "flag" "fmt" "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "time" ) func main() { - var config string - flag.StringVar(&config, "redis", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + var ( + config string + second int + ) + flag.StringVar(&config, "c", "", "config directory") + flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() - if err := internal.Main(config); err != nil { + if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) } } From aff8322bc5ce2929b44666bea03e738bebad7b07 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Tue, 18 Jun 2024 21:04:03 +0800 Subject: [PATCH 054/147] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/friend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 18d80d47d0..b74df1c19b 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -53,7 +53,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"create_time", 1}} + return bson.D{{"is_pinned", -1}, {"create_time", 1}, {"_id", 1}} } // Create inserts multiple friend records. From 08cb4a948d63b3f88dcd5512989e4958d56e5310 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 16:28:28 +0800 Subject: [PATCH 055/147] group --- go.mod | 6 +- go.sum | 4 +- internal/rpc/group/notification.go | 5 ++ internal/rpc/group/sync.go | 54 ++++++++++++-- pkg/common/storage/controller/group.go | 56 +++++++-------- pkg/common/storage/database/group_member.go | 1 + pkg/common/storage/database/mgo/friend.go | 70 +++++++++++++------ .../storage/database/mgo/group_member.go | 6 +- pkg/common/storage/model/friend.go | 18 ++--- 9 files changed, 148 insertions(+), 72 deletions(-) diff --git a/go.mod b/go.mod index 4ab39f7512..d00f0b7395 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.12 + github.com/openimsdk/protocol v0.0.69-alpha.15 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -176,3 +176,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index a26e59d9ed..4c05500bda 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.12 h1:3ZdwmD1y9vcduIC8o2EZS8Ds/fByqcuEFo+NkcBzgRo= -github.com/openimsdk/protocol v0.0.69-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.15 h1:iiVROC2nXTc9qPOSfE7NZckQAVCZiDxBm9yY36ULdyw= +github.com/openimsdk/protocol v0.0.69-alpha.15/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 4fae1ed2b4..5d385d8c14 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -324,6 +324,7 @@ func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -337,6 +338,7 @@ func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } @@ -350,6 +352,7 @@ func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -636,6 +639,7 @@ func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, gr if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMutedNotification, tips) } @@ -663,6 +667,7 @@ func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupCancelMutedNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index bd6fc6399e..10d945a7ea 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -4,10 +4,13 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" + "slices" ) func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { @@ -53,12 +56,36 @@ func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetF } func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + group, err := s.db.TakeGroup(ctx, req.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ - Ctx: ctx, - VersionKey: req.GroupID, - VersionID: req.VersionID, - VersionNumber: req.Version, - Version: s.db.FindMemberIncrVersion, + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit) + if err != nil { + return nil, err + } + var ok bool + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == "" { + ok = true + return true + } + return false + }) + if !ok { + group = nil + } + return vl, nil + }, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) @@ -75,7 +102,22 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou } }, } - return opt.Build() + resp, err := opt.Build() + if err != nil { + return nil, err + } + if group != nil { + count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + resp.Group = s.groupDB2PB(group, owner.UserID, count) + } + return resp, nil } func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index cb64bc73a3..71bf6ead23 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -243,17 +243,10 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { return err } - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { return err } - for _, userID := range userIDs { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate); err != nil { - return err - } - - } - return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxJoinGroupVersion(userIDs...).ChainExecDel(ctx) + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) }) } @@ -263,11 +256,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete if err := g.groupDB.UpdateStatus(ctx, groupID, constant.GroupStatusDismissed); err != nil { return err } - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return err - } if deleteMember { + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } if err := g.groupMemberDB.Delete(ctx, groupID, nil); err != nil { return err } @@ -277,13 +270,18 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). DelGroupMembersInfo(groupID, userIDs...). - DelMaxGroupMemberVersion(groupID) - } - c = c.DelMaxJoinGroupVersion(userIDs...) - if len(userIDs) > 0 { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, model.VersionStateDelete); err != nil { + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...) + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + } else { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { return err } + c = c.DelMaxGroupMemberVersion(groupID) } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) @@ -411,12 +409,16 @@ func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). DelGroupMembersHash(groupID). - DelJoinedGroupID(oldOwnerUserID, newOwnerUserID). + DelMaxGroupMemberVersion(groupID). + DelGroupMemberIDs(groupID). ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { + if len(data) == 0 { + return nil + } return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { return err @@ -424,9 +426,9 @@ func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, u c := g.cache.CloneGroupCache() c = c.DelGroupMembersInfo(groupID, userID) if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) + c = c.DelGroupAllRoleLevel(groupID).DelGroupMemberIDs(groupID) } - c = c.DelMaxGroupMemberVersion(groupID).DelMaxJoinGroupVersion(userID) + c = c.DelMaxGroupMemberVersion(groupID) return c.ChainExecDel(ctx) }) } @@ -439,9 +441,9 @@ func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.B return err } if g.groupMemberDB.IsUpdateRoleLevel(item.Map) { - c = c.DelGroupAllRoleLevel(item.GroupID) + c = c.DelGroupAllRoleLevel(item.GroupID).DelGroupMemberIDs(item.GroupID) } - c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelGroupMembersHash(item.GroupID) + c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelMaxGroupMemberVersion(item.GroupID).DelGroupMembersHash(item.GroupID) } return c.ChainExecDel(ctx) }) @@ -501,14 +503,6 @@ func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) } -//func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { -// return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) -//} -// -//func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { -// return g.cache.FindSortJoinGroupIDs(ctx, userID) -//} - func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { return g.cache.FindMaxGroupMemberVersion(ctx, groupID) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index 0051d694f3..c493196496 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -35,6 +35,7 @@ type GroupMember interface { FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index b74df1c19b..4c3c4b97b9 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -18,6 +18,8 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" @@ -53,11 +55,19 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"create_time", 1}, {"_id", 1}} + return bson.D{{"is_pinned", -1}, {"_id", 1}} } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { + for i, friend := range friends { + if friend.ID.IsZero() { + friends[i].ID = primitive.NewObjectID() + } + if friend.CreateTime.IsZero() { + friends[i].CreateTime = time.Now() + } + } return mongoutil.IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { @@ -108,13 +118,43 @@ func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) } +func (f *FriendMgo) fillTime(friends ...*model.Friend) { + for i, friend := range friends { + if friend.CreateTime.IsZero() { + friends[i].CreateTime = friend.ID.Timestamp() + } + } +} + +func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) { + friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friend) + return friend, nil +} + +func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) { + friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friends...) + return friends, nil +} + +func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) { + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...) +} + // Take retrieves a single friend document. Returns an error if not found. func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) { filter := bson.M{ "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + return f.findOne(ctx, filter) } // FindUserState finds the friendship status between two users. @@ -125,7 +165,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string) {"owner_user_id": userID2, "friend_user_id": userID1}, }, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error. @@ -134,7 +174,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindReversalFriends finds users who have added the specified user as a friend. @@ -143,14 +183,14 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string "owner_user_id": bson.M{"$in": ownerUserIDs}, "friend_user_id": friendUserID, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} opt := options.Find().SetSort(f.friendSort()) - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) + return f.findPage(ctx, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { @@ -162,7 +202,8 @@ func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID stri // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"friend_user_id": friendUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) } // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. @@ -203,18 +244,3 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( } return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } - -//func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// } -// if keyword != "" { -// filter["$or"] = []bson.M{ -// {"remark": bson.M{"$regex": keyword, "$options": "i"}}, -// {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, -// {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, -// } -// } -// opt := options.Find().SetSort(f.friendSort()) -// return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) -//} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 95c6e51a68..99a5a3ed2d 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -117,8 +117,6 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) }, func() error { return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) - }, func() error { - return g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate) }) } @@ -184,6 +182,10 @@ func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string return g.join.IncrVersion(ctx, userID, groupIDs, state) } +func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + return g.member.IncrVersion(ctx, groupID, userIDs, state) +} + func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { return g.member.FindChangeLog(ctx, groupID, version, limit) } diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2a..abcca2f2b9 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -15,17 +15,19 @@ package model import ( + "go.mongodb.org/mongo-driver/bson/primitive" "time" ) // Friend represents the data structure for a friend relationship in MongoDB. type Friend struct { - OwnerUserID string `bson:"owner_user_id"` - FriendUserID string `bson:"friend_user_id"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` + ID primitive.ObjectID `bson:"_id"` + OwnerUserID string `bson:"owner_user_id"` + FriendUserID string `bson:"friend_user_id"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` } From f613c6dfc9933e817e2b96a8e8831eb23543a75c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 16:52:44 +0800 Subject: [PATCH 056/147] group --- internal/rpc/group/sync.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 10d945a7ea..3793d0c7e6 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -63,6 +63,7 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } + var hasGroupUpdate bool opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, VersionKey: req.GroupID, @@ -73,17 +74,14 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - var ok bool vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { if elem.EID == "" { - ok = true + vl.LogLen-- + hasGroupUpdate = true return true } return false }) - if !ok { - group = nil - } return vl, nil }, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, @@ -106,7 +104,7 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - if group != nil { + if resp.Full || hasGroupUpdate { count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err From 1ee33f36f71dba904cfe8dd75d02a84fcd347743 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 17:38:30 +0800 Subject: [PATCH 057/147] group --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/group/notification.go | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d00f0b7395..34987958fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.15 + github.com/openimsdk/protocol v0.0.69-alpha.16 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 4c05500bda..3ab5684a93 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.15 h1:iiVROC2nXTc9qPOSfE7NZckQAVCZiDxBm9yY36ULdyw= -github.com/openimsdk/protocol v0.0.69-alpha.15/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.16 h1:ciSqm2rjBdpScpkQm3wPjAFv0YbIRp8MITRkDZWVv6c= +github.com/openimsdk/protocol v0.0.69-alpha.16/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 5d385d8c14..ee4f00ad36 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -477,11 +477,16 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. } opUserID := mcontext.GetOpUserID(ctx) var member map[string]*sdkws.GroupMemberFullInfo - member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID}) + member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID, req.OldOwnerUserID}) if err != nil { return } - tips := &sdkws.GroupOwnerTransferredTips{Group: group, OpUser: member[opUserID], NewGroupOwner: member[req.NewOwnerUserID]} + tips := &sdkws.GroupOwnerTransferredTips{ + Group: group, + OpUser: member[opUserID], + NewGroupOwner: member[req.NewOwnerUserID], + OldGroupOwnerInfo: member[req.OldOwnerUserID], + } if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } From 924888dbd125de298cb2c4ea74d4307d21f57123 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:17:28 +0800 Subject: [PATCH 058/147] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/version_log.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 1d70f96f51..8836742f08 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -8,6 +8,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -169,7 +170,9 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } + log.ZDebug(ctx, "init doc", "dId", dId) if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dId) return res, nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) From 5006af5b54ba47db5ada3b06b1298836dc5ee064 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:28:09 +0800 Subject: [PATCH 059/147] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/group_member.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 99a5a3ed2d..30f8d63b9f 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/log" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" @@ -187,9 +188,11 @@ func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID str } func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find member incr version", "groupID", groupID, "version", version) return g.member.FindChangeLog(ctx, groupID, version, limit) } func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version) return g.join.FindChangeLog(ctx, userID, version, limit) } From 4f359b733556290ca137b4529bc74bd7ea11c59d Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:35:32 +0800 Subject: [PATCH 060/147] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/cache/redis/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index c4598567d3..589678c506 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -446,7 +446,7 @@ func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupC func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { - return g.groupMemberDB.FindJoinIncrVersion(ctx, groupID, 0, 0) + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, 0, 0) }) } From 03d4564596c746f81ab1e95e61a3cc6532aeace1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 20 Jun 2024 14:55:59 +0800 Subject: [PATCH 061/147] user version --- go.mod | 6 +- go.sum | 4 +- internal/rpc/friend/sync.go | 16 +++ internal/rpc/group/group.go | 6 +- internal/rpc/user/user.go | 122 +++++++++++++++------- pkg/common/storage/controller/friend.go | 9 +- pkg/common/storage/controller/group.go | 8 ++ pkg/common/storage/database/friend.go | 2 + pkg/common/storage/database/mgo/friend.go | 4 + 9 files changed, 129 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 34987958fb..2614e0f322 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.16 + github.com/openimsdk/protocol v0.0.69-alpha.17 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -177,6 +177,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 3ab5684a93..fe4f0c390a 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.16 h1:ciSqm2rjBdpScpkQm3wPjAFv0YbIRp8MITRkDZWVv6c= -github.com/openimsdk/protocol v0.0.69-alpha.16/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= +github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index faaa987f20..684894609c 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -11,6 +11,22 @@ import ( "github.com/openimsdk/protocol/relation" ) +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(ctx, userID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(ctx, req.UserID, userID) + } + return &relation.NotificationUserInfoUpdateResp{}, nil +} + func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) if err != nil { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 4d429b3d3b..5b82b8ceb2 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -132,13 +132,17 @@ func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro } groupIDs = append(groupIDs, member.GroupID) } + for _, groupID := range groupIDs { + if err := s.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } for _, groupID := range groupIDs { s.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) } if err = s.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { return nil, err } - return &pbgroup.NotificationUserInfoUpdateResp{}, nil } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 96f784fab0..211b360b7b 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,10 +16,7 @@ package user import ( "context" - "math/rand" - "strings" - "time" - + "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -27,7 +24,13 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/protocol/group" + friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" + "math/rand" + "strings" + "sync" + "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -120,7 +123,8 @@ func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesig // deprecated: -// UpdateUserInfo +//UpdateUserInfo + func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -131,31 +135,33 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } - func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -165,30 +171,33 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - } + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } - func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { resp = &pbuser.SetGlobalRecvMessageOptResp{} if _, err := s.db.FindWithError(ctx, []string{req.UserID}); err != nil { @@ -247,6 +256,7 @@ func (s *userServer) GetPaginationUsers(ctx context.Context, req *pbuser.GetPagi return &pbuser.GetPaginationUsersResp{Total: int32(total), Users: convert.UsersDB2Pb(users)}, err } + } func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterReq) (resp *pbuser.UserRegisterResp, err error) { @@ -343,8 +353,7 @@ func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbus // GetUserStatus Get the online status of the user. func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error, -) { + err error) { onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) if err != nil { return nil, err @@ -354,8 +363,7 @@ func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatu // SetUserStatus Synchronize user's online status. func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error, -) { + err error) { err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) if err != nil { return nil, err @@ -379,8 +387,7 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq, -) (*pbuser.GetSubscribeUsersStatusResp, error) { + req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) if err != nil { return nil, err @@ -469,6 +476,7 @@ func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.P } func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.ProcessUserCommandGetReq) (*pbuser.ProcessUserCommandGetResp, error) { + err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err @@ -687,6 +695,40 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} + func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) if err != nil { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 42c2305988..4d3a96f12e 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -86,7 +86,7 @@ type FriendDatabase interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } type friendDatabase struct { @@ -379,3 +379,10 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri //func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { // return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) //} + +func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + if err := f.friend.IncrVersion(ctx, ownerUserID, friendUserIDs, state); err != nil { + return err + } + return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 71bf6ead23..27b6f43662 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -109,6 +109,7 @@ type GroupDatabase interface { FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) @@ -518,3 +519,10 @@ func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyw } return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) } + +func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, userIDs, state); err != nil { + return err + } + return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 1c9cd10330..b596411fce 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -55,4 +55,6 @@ type Friend interface { //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) + + IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 4c3c4b97b9..7f456fbdab 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -244,3 +244,7 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( } return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } + +func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) +} From 2d2941a52645d1bd104054d87b20dbbca862c477 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 20 Jun 2024 15:56:02 +0800 Subject: [PATCH 062/147] seq --- pkg/common/storage/controller/msg.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3a29f6786c..de8c111b3d 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -340,11 +340,6 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) - if err != nil { - log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err - } lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() @@ -352,6 +347,12 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa if lenList < 1 { return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() } + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err + } + isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) for _, m := range msgs { From 8e3890ed8824e5e61994adc50b8253457db7c0af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 21 Jun 2024 18:24:30 +0800 Subject: [PATCH 063/147] seq --- pkg/common/storage/cache/redis/seq.go | 64 +++++++++++-------- pkg/common/storage/database/mgo/seq_user.go | 71 +++++++++++++++++++++ pkg/common/storage/database/name.go | 1 + pkg/common/storage/database/seq_user.go | 12 ++++ pkg/common/storage/model/seq_user.go | 9 +++ 5 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 pkg/common/storage/database/mgo/seq_user.go create mode 100644 pkg/common/storage/database/seq_user.go create mode 100644 pkg/common/storage/model/seq_user.go diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go index 76dd921a50..8624cc78fa 100644 --- a/pkg/common/storage/cache/redis/seq.go +++ b/pkg/common/storage/cache/redis/seq.go @@ -75,21 +75,43 @@ func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s st return m, nil } -func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -} - -func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -} +//func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { +// return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) +//} +// +//func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { +// return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) +//} +// +//func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +// return c.getSeq(ctx, conversationID, c.getMaxSeqKey) +//} +// +//func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) +//} +// +//func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { +// for conversationID, seq := range seqs { +// if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { +// return errs.Wrap(err) +// } +// } +// return nil +//} +// +// +//func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { +// return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) +//} +// +//func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +// return c.getSeq(ctx, conversationID, c.getMinSeqKey) +//} +// +//func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { +// return c.setSeqs(ctx, seqs, c.getMinSeqKey) +//} func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { for conversationID, seq := range seqs { @@ -100,18 +122,6 @@ func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey fu return nil } -func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return c.setSeqs(ctx, seqs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMinSeqKey) -} - func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() if err != nil { diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go new file mode 100644 index 0000000000..4fa299a7a6 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -0,0 +1,71 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "user_id", Value: 1}, + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqUserMongo{coll: coll}, nil +} + +type seqUserMongo struct { + coll *mongo.Collection +} + +func (s *seqUserMongo) setSeq(ctx context.Context, userID string, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + update := map[string]any{ + "$set": map[string]any{"field": int64(0)}, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqUserMongo) GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 68fa5af02c..f47b7a6ee8 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -15,4 +15,5 @@ const ( ObjectName = "s3" UserName = "user" SeqConversationName = "seq" + SeqUserName = "seq_user" ) diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go new file mode 100644 index 0000000000..3e9b29ec20 --- /dev/null +++ b/pkg/common/storage/database/seq_user.go @@ -0,0 +1,12 @@ +package database + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/model/seq_user.go b/pkg/common/storage/model/seq_user.go new file mode 100644 index 0000000000..845996bb8b --- /dev/null +++ b/pkg/common/storage/model/seq_user.go @@ -0,0 +1,9 @@ +package model + +type SeqUser struct { + UserID string `bson:"user_id"` + ConversationID string `bson:"conversation_id"` + MinSeq int64 `bson:"min_seq"` + MaxSeq int64 `bson:"max_seq"` + ReadSeq int64 `bson:"read_seq"` +} From 9b043feca7aeb5abe21c7c4bc996da5d04f2d446 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 24 Jun 2024 17:34:49 +0800 Subject: [PATCH 064/147] seq user --- internal/msgtransfer/init.go | 8 +- internal/rpc/msg/server.go | 8 +- pkg/common/storage/cache/cachekey/seq.go | 44 +-- pkg/common/storage/cache/cachekey/seq1.go | 14 - pkg/common/storage/cache/redis/seq.go | 183 ---------- pkg/common/storage/cache/redis/seq_user.go | 89 +++++ pkg/common/storage/cache/seq.go | 30 -- pkg/common/storage/cache/seq_user.go | 12 + pkg/common/storage/controller/msg.go | 74 ++-- .../storage/database/mgo/seq_conversation.go | 34 +- .../database/mgo/seq_conversation_test.go | 34 +- pkg/common/storage/database/mgo/seq_user.go | 63 ++-- pkg/common/storage/database/seq.go | 1 + pkg/common/storage/database/seq_user.go | 12 +- tools/seq/internal/main.go | 315 ++++++++++++------ tools/seq/main.go | 3 +- 16 files changed, 482 insertions(+), 442 deletions(-) delete mode 100644 pkg/common/storage/cache/cachekey/seq1.go delete mode 100644 pkg/common/storage/cache/redis/seq.go create mode 100644 pkg/common/storage/cache/redis/seq_user.go delete mode 100644 pkg/common/storage/cache/seq.go create mode 100644 pkg/common/storage/cache/seq_user.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index e90c2288c8..c17abbc305 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -82,7 +82,6 @@ func Start(ctx context.Context, index int, config *Config) error { client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err @@ -92,7 +91,12 @@ func Start(ctx context.Context, index int, config *Config) error { return err } seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 7cffb23a9c..de0f698ea3 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -86,7 +86,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) @@ -96,7 +95,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index ded0286d84..b32e78300c 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -1,38 +1,30 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cachekey const ( - MaxSeq = "MAX_SEQ:" - MinSeq = "MIN_SEQ:" - ConversationUserMinSeq = "CON_USER_MIN_SEQ:" - HasReadSeq = "HAS_READ_SEQ:" + MallocSeq = "MALLOC_SEQ:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" + + SeqUserMaxSeq = "SEQ_USER_MAX:" + SeqUserMinSeq = "SEQ_USER_MIN:" + SeqUserReadSeq = "SEQ_USER_READ:" ) -func GetMaxSeqKey(conversationID string) string { - return MaxSeq + conversationID +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID } -func GetMinSeqKey(conversationID string) string { - return MinSeq + conversationID +func GetSeqUserMaxSeqKey(conversationID string, userID string) string { + return SeqUserMaxSeq + conversationID + ":" + userID } -func GetHasReadSeqKey(conversationID string, userID string) string { - return HasReadSeq + userID + ":" + conversationID +func GetSeqUserMinSeqKey(conversationID string, userID string) string { + return SeqUserMinSeq + conversationID + ":" + userID } -func GetConversationUserMinSeqKey(conversationID, userID string) string { - return ConversationUserMinSeq + conversationID + "u:" + userID +func GetSeqUserReadSeqKey(conversationID string, userID string) string { + return SeqUserReadSeq + conversationID + ":" + userID } diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go deleted file mode 100644 index 04274db55a..0000000000 --- a/pkg/common/storage/cache/cachekey/seq1.go +++ /dev/null @@ -1,14 +0,0 @@ -package cachekey - -const ( - MallocSeq = "MALLOC_SEQ:" - MallocMinSeqLock = "MALLOC_MIN_SEQ:" -) - -func GetMallocSeqKey(conversationID string) string { - return MallocSeq + conversationID -} - -func GetMallocMinSeqKey(conversationID string) string { - return MallocMinSeqLock + conversationID -} diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index 8624cc78fa..0000000000 --- a/pkg/common/storage/cache/redis/seq.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redis - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" - "github.com/redis/go-redis/v9" -) - -func NewSeqCache(rdb redis.UniversalClient) cache.SeqCache { - return &seqCache{rdb: rdb} -} - -type seqCache struct { - rdb redis.UniversalClient -} - -func (c *seqCache) getMaxSeqKey(conversationID string) string { - return cachekey.GetMaxSeqKey(conversationID) -} - -func (c *seqCache) getMinSeqKey(conversationID string) string { - return cachekey.GetMinSeqKey(conversationID) -} - -func (c *seqCache) getHasReadSeqKey(conversationID string, userID string) string { - return cachekey.GetHasReadSeqKey(conversationID, userID) -} - -func (c *seqCache) getConversationUserMinSeqKey(conversationID, userID string) string { - return cachekey.GetConversationUserMinSeqKey(conversationID, userID) -} - -func (c *seqCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error { - return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err()) -} - -func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey func(conversationID string) string) (int64, error) { - val, err := c.rdb.Get(ctx, getkey(conversationID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) { - m = make(map[string]int64, len(items)) - for i, v := range items { - res, err := c.rdb.Get(ctx, getkey(v)).Result() - if err != nil && err != redis.Nil { - return nil, errs.Wrap(err) - } - val := stringutil.StringToInt64(res) - if val != 0 { - m[items[i]] = val - } - } - - return m, nil -} - -//func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { -// return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -//} -// -//func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { -// return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -//} -// -//func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { -// return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -//} -// -//func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { -// return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -//} -// -//func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { -// for conversationID, seq := range seqs { -// if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { -// return errs.Wrap(err) -// } -// } -// return nil -//} -// -// -//func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { -// return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -//} -// -//func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { -// return c.getSeq(ctx, conversationID, c.getMinSeqKey) -//} -// -//func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { -// return c.setSeqs(ctx, seqs, c.getMinSeqKey) -//} - -func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { - for conversationID, seq := range seqs { - if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { - return errs.Wrap(err) - } - } - return nil -} - -func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, userIDs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getConversationUserMinSeqKey(conversationID, userID), minSeq, 0).Err()) -} - -func (c *seqCache) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(conversationID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getHasReadSeqKey(conversationID, userID), hasReadSeq, 0).Err()) -} - -func (c *seqCache) SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(userID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getHasReadSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, err - } - return val, nil -} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go new file mode 100644 index 0000000000..3c533cdd83 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -0,0 +1,89 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +func NewSeqUserCacheRedis(rdb redis.UniversalClient, mgo database.SeqUser) cache.SeqUser { + return &seqUserCacheRedis{ + rdb: rdb, + mgo: mgo, + readSeqWriteRatio: 100, + expireTime: time.Hour * 24 * 7, + readExpireTime: time.Hour * 24 * 30, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqUserCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqUser + rocks *rockscache.Client + expireTime time.Duration + readExpireTime time.Duration + readSeqWriteRatio int64 +} + +func (s *seqUserCacheRedis) getSeqUserMaxSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMaxSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserMinSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMinSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserReadSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetMaxSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMinSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if seq%s.readSeqWriteRatio == 0 { + if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + return errs.Wrap(err) + } + return nil +} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go deleted file mode 100644 index 46e33c9357..0000000000 --- a/pkg/common/storage/cache/seq.go +++ /dev/null @@ -1,30 +0,0 @@ -package cache - -import ( - "context" -) - -type SeqCache interface { - //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - //GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - //GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - //SetMinSeqs(ctx context.Context, seqs map[string]int64) error - //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - //GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - // seqs map: key userID value minSeq - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) - // seqs map: key conversationID value minSeq - SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - // has read seq - SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - // k: user, v: seq - SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error - // k: conversation, v :seq - UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) - GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) -} diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go new file mode 100644 index 0000000000..9e68399a73 --- /dev/null +++ b/pkg/common/storage/cache/seq_user.go @@ -0,0 +1,12 @@ +package cache + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index de8c111b3d..1e7b5c2290 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -108,7 +108,7 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -128,7 +128,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, - seq: seq, + seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, producerToMongo: producerToMongo, @@ -140,8 +140,8 @@ type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache - seq cache.SeqCache seqConversation cache.SeqConversationCache + seqUser cache.SeqUser producer *kafka.Producer producerToMongo *kafka.Producer producerToPush *kafka.Producer @@ -339,6 +339,15 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) } +func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil +} + func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { @@ -368,7 +377,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) if err != nil { log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) prommetrics.SeqSetFailedCounter.Inc() @@ -496,7 +505,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -574,7 +583,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -672,12 +681,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + currentUserMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -832,40 +841,45 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int return nil } -func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) { - return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs) -} - func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs) + for conversationID, seq := range seqs { + if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs) + for conversationID, seq := range hasReadSeqs { + if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq) + return db.seqUser.SetReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs) + cSeq := make(map[string]int64) + for _, conversationID := range conversationIDs { + if _, ok := cSeq[conversationID]; ok { + continue + } + seq, err := db.seqUser.GetReadSeq(ctx, conversationID, userID) + if err != nil { + return nil, err + } + cSeq[conversationID] = seq + } + return cSeq, nil } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seq.GetHasReadSeq(ctx, userID, conversationID) + return db.seqUser.GetReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { diff --git a/pkg/common/storage/database/mgo/seq_conversation.go b/pkg/common/storage/database/mgo/seq_conversation.go index c9a3ac41a9..7971b7e1a7 100644 --- a/pkg/common/storage/database/mgo/seq_conversation.go +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -28,6 +28,26 @@ type seqConversationMongo struct { coll *mongo.Collection } +func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "conversation_id": conversationID, + } + insert := bson.M{ + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errors.New("size must be greater than 0") @@ -48,16 +68,8 @@ func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string return lastSeq - size, nil } -func (s *seqConversationMongo) MallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { - first, err := s.Malloc(ctx, conversationID, size) - if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := int64(0); i < size; i++ { - seqs = append(seqs, first+i+1) - } - return seqs, nil +func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "max_seq") } func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { @@ -83,7 +95,7 @@ func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID str } func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { - return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) + return s.setSeq(ctx, conversationID, seq, "min_seq") } func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index e6466a1c60..5167314da9 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -15,19 +15,23 @@ func Result[V any](val V, err error) V { return val } -func TestName(t *testing.T) { - cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - tmp, err := NewSeqConversationMongo(cli.Database("openim_v3")) - if err != nil { - panic(err) - } - for i := 0; i < 10; i++ { - var size int64 = 100 - firstSeq, err := tmp.Malloc(context.Background(), "1", size) - if err != nil { - t.Log(err) - return - } - t.Logf("%d -> %d", firstSeq, firstSeq+size-1) - } +func Mongodb() *mongo.Database { + return Result( + mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)), + ).Database("openim_v3") +} + +func TestUserSeq(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4)) +} + +func TestConversationSeq(t *testing.T) { + cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo) + t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10)) + t.Log(cSeq.Malloc(context.Background(), "2000", 10)) + t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) } diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 4fa299a7a6..5e0eb2022d 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -2,6 +2,7 @@ package mgo import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" @@ -10,7 +11,7 @@ import ( ) func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { - coll := db.Collection(database.SeqConversationName) + coll := db.Collection(database.SeqUserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, @@ -27,45 +28,65 @@ type seqUserMongo struct { coll *mongo.Collection } -func (s *seqUserMongo) setSeq(ctx context.Context, userID string, conversationID string, seq int64, field string) error { +func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error { filter := map[string]any{ "user_id": userID, "conversation_id": conversationID, } + insert := bson.M{ + "user_id": userID, + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + "read_seq": 0, + } + delete(insert, field) update := map[string]any{ - "$set": map[string]any{"field": int64(0)}, + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, } opt := options.Update().SetUpsert(true) return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) } -func (s *seqUserMongo) GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) { +func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1}) + seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt) + if err == nil { + return seq, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return 0, nil + } else { + return 0, err + } +} - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "max_seq") } -func (s *seqUserMongo) SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "max_seq") } -func (s *seqUserMongo) GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "min_seq") } -func (s *seqUserMongo) SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "min_seq") } -func (s *seqUserMongo) GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "read_seq") } -func (s *seqUserMongo) SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index fdbc2f8f33..cf93b795f4 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -5,6 +5,7 @@ import "context" type SeqConversation interface { Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, seq int64) error GetMinSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeq(ctx context.Context, conversationID string, seq int64) error } diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go index 3e9b29ec20..d32d614ddd 100644 --- a/pkg/common/storage/database/seq_user.go +++ b/pkg/common/storage/database/seq_user.go @@ -3,10 +3,10 @@ package database import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error - GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error - GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error } diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 54a7d477da..5200ad5fef 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -1,30 +1,38 @@ package internal import ( + "bytes" "context" "errors" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "gopkg.in/yaml.v3" "os" + "os/signal" "path/filepath" "strconv" "strings" + "sync" + "sync/atomic" + "syscall" "time" ) +const ( + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" +) + const ( batchSize = 100 dataVersionCollection = "data_version" @@ -44,48 +52,6 @@ func readConfig[T any](dir string, name string) (*T, error) { return &conf, nil } -func redisKey(rdb redis.UniversalClient, prefix string, del time.Duration, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { - var ( - cursor uint64 - keys []string - err error - ) - ctx := context.Background() - for { - keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() - if err != nil { - return err - } - delKey := make(map[string]struct{}) - if len(keys) > 0 { - for _, key := range keys { - if err := fn(ctx, key, delKey); err != nil { - return err - } - } - } - if len(delKey) > 0 { - delKeys := datautil.Keys(delKey) - if del < time.Second { - if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { - return err - } - } else { - pipe := rdb.Pipeline() - for _, key := range delKeys { - pipe.Expire(ctx, key, del) - } - if _, err := pipe.Exec(ctx); err != nil { - return err - } - } - } - if cursor == 0 { - return nil - } - } -} - func Main(conf string, del time.Duration) error { redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) if err != nil { @@ -117,71 +83,218 @@ func Main(conf string, del time.Duration) error { if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { return err } - coll := mgocli.GetDB().Collection(database.SeqConversationName) - const prefix = cachekey.MaxSeq - fmt.Println("start to convert seq conversation") - err = redisKey(rdb, prefix, del, func(ctx context.Context, key string, delKey map[string]struct{}) error { - conversationId := strings.TrimPrefix(key, prefix) - delKey[key] = struct{}{} - maxValue, err := rdb.Get(ctx, key).Result() - if err != nil { - return err - } - seq, err := strconv.Atoi(maxValue) - if err != nil { - return fmt.Errorf("invalid max seq %s", maxValue) + cSeq, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSeq, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSpitHasReadSeq := func(id string) (conversationID string, userID string, err error) { + // HasReadSeq + userID + ":" + conversationID + arr := strings.Split(id, ":") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) } - if seq == 0 { - return nil + userID = arr[0] + conversationID = arr[1] + return + } + uSpitConversationUserMinSeq := func(id string) (conversationID string, userID string, err error) { + // ConversationUserMinSeq + conversationID + "u:" + userID + arr := strings.Split(id, "u:") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) } - if seq < 0 { - return fmt.Errorf("invalid max seq %s", maxValue) + conversationID = arr[0] + userID = arr[1] + return + } + + ts := []*taskSeq{ + { + Prefix: MaxSeq, + GetSeq: cSeq.GetMaxSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: MinSeq, + GetSeq: cSeq.GetMinSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: HasReadSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetReadSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return err + } + return uSeq.SetReadSeq(ctx, conversationID, userID, seq) + }, + }, + { + Prefix: ConversationUserMinSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetMinSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return err + } + return uSeq.SetMinSeq(ctx, conversationID, userID, seq) + }, + }, + } + + cancel() + ctx = context.Background() + + var wg sync.WaitGroup + wg.Add(len(ts)) + + for i := range ts { + go func(task *taskSeq) { + defer wg.Done() + err := seqRedisToMongo(ctx, rdb, task.GetSeq, task.SetSeq, task.Prefix, del, &task.Count) + task.End = time.Now() + task.Error = err + }(ts[i]) + } + start := time.Now() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var buf bytes.Buffer + + printTaskInfo := func(now time.Time) { + buf.Reset() + buf.WriteString(now.Format(time.DateTime)) + buf.WriteString(" \n") + for i := range ts { + task := ts[i] + if task.Error == nil { + if task.End.IsZero() { + buf.WriteString(fmt.Sprintf("[%s] converting %s* count %d", now.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } else { + buf.WriteString(fmt.Sprintf("[%s] success %s* count %d", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } + } else { + buf.WriteString(fmt.Sprintf("[%s] failed %s* count %d error %s", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count), task.Error)) + } + buf.WriteString("\n") } - var ( - minSeq int64 - maxSeq = int64(seq) - ) - minKey := cachekey.MinSeq + conversationId - delKey[minKey] = struct{}{} - minValue, err := rdb.Get(ctx, minKey).Result() - if err == nil { - seq, err := strconv.Atoi(minValue) - if err != nil { - return fmt.Errorf("invalid min seq %s", minValue) + fmt.Println(buf.String()) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case s := <-sigs: + return fmt.Errorf("exit by signal %s", s) + case <-done: + errs := make([]error, 0, len(ts)) + for i := range ts { + task := ts[i] + if task.Error != nil { + errs = append(errs, fmt.Errorf("seq %s failed %w", task.Prefix, task.Error)) + } } - if seq < 0 { - return fmt.Errorf("invalid min seq %s", minValue) + if len(errs) > 0 { + return errors.Join(errs...) } - minSeq = int64(seq) - } else if !errors.Is(err, redis.Nil) { - return err - } - if maxSeq < minSeq { - return fmt.Errorf("invalid max seq %d < min seq %d", maxSeq, minSeq) - } - res, err := mongoutil.FindOne[*model.SeqConversation](ctx, coll, bson.M{"conversation_id": conversationId}, nil) - if err == nil { - if res.MaxSeq < int64(seq) { - _, err = coll.UpdateOne(ctx, bson.M{"conversation_id": conversationId}, bson.M{"$set": bson.M{"max_seq": maxSeq, "min_seq": minSeq}}) + printTaskInfo(time.Now()) + if err := SetVersion(versionColl, seqKey, seqVersion); err != nil { + return fmt.Errorf("set mongodb seq version %w", err) } + return nil + case now := <-ticker.C: + printTaskInfo(now) + } + } +} + +type taskSeq struct { + Prefix string + Count int64 + Error error + End time.Time + GetSeq func(ctx context.Context, id string) (int64, error) + SetSeq func(ctx context.Context, id string, seq int64) error +} + +func seqRedisToMongo(ctx context.Context, rdb redis.UniversalClient, getSeq func(ctx context.Context, id string) (int64, error), setSeq func(ctx context.Context, id string, seq int64) error, prefix string, delAfter time.Duration, count *int64) error { + var ( + cursor uint64 + keys []string + err error + ) + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { return err - } else if errors.Is(err, mongo.ErrNoDocuments) { - res = &model.SeqConversation{ - ConversationID: conversationId, - MaxSeq: maxSeq, - MinSeq: minSeq, + } + if len(keys) > 0 { + for _, key := range keys { + seqStr, err := rdb.Get(ctx, key).Result() + if err != nil { + return fmt.Errorf("redis get %s failed %w", key, err) + } + seq, err := strconv.Atoi(seqStr) + if err != nil { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + if seq < 0 { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + id := strings.TrimPrefix(key, prefix) + redisSeq := int64(seq) + mongoSeq, err := getSeq(ctx, id) + if err != nil { + return fmt.Errorf("get mongo seq %s failed %w", key, err) + } + if mongoSeq < redisSeq { + if err := setSeq(ctx, id, redisSeq); err != nil { + return fmt.Errorf("set mongo seq %s failed %w", key, err) + } + } + if delAfter > 0 { + if err := rdb.Expire(ctx, key, delAfter).Err(); err != nil { + return fmt.Errorf("redis expire key %s failed %w", key, err) + } + } else { + if err := rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("redis del key %s failed %w", key, err) + } + } + atomic.AddInt64(count, 1) } - _, err := coll.InsertOne(ctx, res) - return err - } else { - return err } - }) - if err != nil { - return err + if cursor == 0 { + return nil + } } - fmt.Println("convert seq conversation success") - return SetVersion(versionColl, seqKey, seqVersion) } func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { diff --git a/tools/seq/main.go b/tools/seq/main.go index 6bb4b76578..399e6d934e 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -12,10 +12,11 @@ func main() { config string second int ) - flag.StringVar(&config, "c", "", "config directory") + flag.StringVar(&config, "c", "/Users/chao/Desktop/project/open-im-server/config", "config directory") flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) } + fmt.Println("seq task success!") } From 8f0403e67c2415d0cc0760e695ba663caecfca76 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 26 Jun 2024 18:19:38 +0800 Subject: [PATCH 065/147] user online --- .../openim-rpc-conversation/main.go | 4 ++ cmd/openim-rpc/openim-rpc-group/main.go | 4 ++ config/discovery.yml | 4 +- config/kafka.yml | 2 +- config/minio.yml | 4 +- config/mongodb.yml | 2 +- config/redis.yml | 2 +- internal/msggateway/n_ws_server.go | 6 +- internal/msggateway/user_map.go | 4 +- internal/msggateway/user_map2.go | 32 +++++++++ pkg/common/storage/cache/redis/online.go | 58 +++++++++++++++ .../storage/cache/redis/seq_user_test.go | 70 +++++++++++++++++++ pkg/common/storage/cache/redis/user.go | 9 +++ 13 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 internal/msggateway/user_map2.go create mode 100644 pkg/common/storage/cache/redis/online.go create mode 100644 pkg/common/storage/cache/redis/seq_user_test.go diff --git a/cmd/openim-rpc/openim-rpc-conversation/main.go b/cmd/openim-rpc/openim-rpc-conversation/main.go index 5b2e66c954..5b3077ccb2 100644 --- a/cmd/openim-rpc/openim-rpc-conversation/main.go +++ b/cmd/openim-rpc/openim-rpc-conversation/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewConversationRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/cmd/openim-rpc/openim-rpc-group/main.go b/cmd/openim-rpc/openim-rpc-group/main.go index 5badf934e9..44e5509df1 100644 --- a/cmd/openim-rpc/openim-rpc-group/main.go +++ b/cmd/openim-rpc/openim-rpc-group/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewGroupRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/config/discovery.yml b/config/discovery.yml index 3d96ff9b66..9cd23c590f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,13 +1,13 @@ enable: "etcd" etcd: rootDirectory: openim - address: [ localhost:12379 ] + address: [ 172.16.8.48:12379 ] username: '' password: '' zookeeper: schema: openim - address: [ localhost:12181 ] + address: [ 172.16.8.48:12181 ] username: '' password: '' diff --git a/config/kafka.yml b/config/kafka.yml index d412e1be06..d9b7ffa3cf 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -7,7 +7,7 @@ producerAck: "" # Compression type to use (e.g., none, gzip, snappy) compressType: "none" # List of Kafka broker addresses -address: [ localhost:19094 ] +address: [ 172.16.8.48:19094 ] # Kafka topic for Redis integration toRedisTopic: "toRedis" # Kafka topic for MongoDB integration diff --git a/config/minio.yml b/config/minio.yml index 11a9ace354..d143a1da38 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -7,9 +7,9 @@ secretAccessKey: "openIM123" # Session token for MinIO authentication (optional) sessionToken: '' # Internal address of the MinIO server -internalAddress: "localhost:10005" +internalAddress: "172.16.8.48:10005" # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://external_ip:10005" +externalAddress: "http://172.16.8.48:10005" # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 98f5694e45..53969298b4 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,7 +1,7 @@ # URI for database connection, leave empty if using address and credential settings directly uri: '' # List of MongoDB server addresses -address: [ localhost:37017 ] +address: [ 172.16.8.48:37017 ] # Name of the database database: openim_v3 # Username for database authentication diff --git a/config/redis.yml b/config/redis.yml index 87abed0e1c..404d18953f 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ localhost:16379 ] +address: [ 172.16.8.48:16379 ] username: '' password: openIM123 clusterMode: false diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index defec16df1..17e550fefc 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -309,7 +309,7 @@ func getRemoteAdders(client []*Client) string { } func (ws *WsServer) KickUserConn(client *Client) error { - ws.clients.deleteClients(client.UserID, []*Client{client}) + ws.clients.DeleteClients(client.UserID, []*Client{client}) return client.KickOnlineMessage() } @@ -325,7 +325,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if !clientOK { return } - ws.clients.deleteClients(newClient.UserID, oldClients) + ws.clients.DeleteClients(newClient.UserID, oldClients) for _, c := range oldClients { err := c.KickOnlineMessage() if err != nil { @@ -345,7 +345,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.Delete(client.UserID, client.ctx.GetRemoteAddr()) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 79cc53d1bc..f8bf69f9ad 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -72,7 +72,7 @@ func (u *UserMap) Set(key string, v *Client) { } } -func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) { +func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { // Attempt to load the clients associated with the key. allClients, existed := u.m.Load(key) if !existed { @@ -101,7 +101,7 @@ func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) return false } -func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser bool) { +func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { return c.ctx.GetRemoteAddr(), struct{}{} }) diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go new file mode 100644 index 0000000000..1c9a91c6a6 --- /dev/null +++ b/internal/msggateway/user_map2.go @@ -0,0 +1,32 @@ +package msggateway + +/* + +sorted set + +userID: 10000 + +USER_ONLINE:10000 + + + + + +platformID: 1 + + + + + +key score + +E1 123456789 +O1 234567895 + + + + + + + +*/ diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go new file mode 100644 index 0000000000..138f9a5738 --- /dev/null +++ b/pkg/common/storage/cache/redis/online.go @@ -0,0 +1,58 @@ +package redis + +import ( + "context" + "github.com/redis/go-redis/v9" + "time" +) + +type userOnline struct { + rdb redis.UniversalClient + expire time.Duration + channelName string +} + +func (s *userOnline) getUserOnlineKey(userID string) string { + return "USER_ONLINE:" + userID +} + +func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { + script := ` + local key = KEYS[1] + local score = ARGV[3] + local num1 = redis.call("ZCARD", key) + redis.call("ZREMRANGEBYSCORE", key, "-inf", ARGV[2]) + for i = 5, tonumber(ARGV[4])+4 do + redis.call("ZREM", key, ARGV[i]) + end + local num2 = redis.call("ZCARD", key) + for i = 5+tonumber(ARGV[4]), #ARGV do + redis.call("ZADD", key, score, ARGV[i]) + end + redis.call("EXPIRE", key, ARGV[1]) + local num3 = redis.call("ZCARD", key) + local change = (num1 ~= num2) or (num2 ~= num3) + if change then + local members = redis.call("ZRANGE", key, 0, -1) + table.insert(members, KEYS[2]) + redis.call("PUBLISH", KEYS[3], table.concat(members, ":")) + return 1 + else + return 0 + end +` + now := time.Now() + argv := make([]any, 0, 2+len(online)+len(offline)) + argv = append(argv, int32(s.expire/time.Second), now.Unix(), now.Add(s.expire).Unix(), int32(len(offline))) + for _, platformID := range offline { + argv = append(argv, platformID) + } + for _, platformID := range online { + argv = append(argv, platformID) + } + keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} + if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + return err + } + return nil +} diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go new file mode 100644 index 0000000000..04a5d49cb1 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -0,0 +1,70 @@ +package redis + +import ( + "context" + "fmt" + "github.com/redis/go-redis/v9" + "log" + "testing" + "time" +) + +func newTestOnline() *userOnline { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &userOnline{rdb: rdb, expire: time.Hour, channelName: "user_online"} +} + +func TestOnline(t *testing.T) { + ts := newTestOnline() + + //err := ts.SetUserOnline(context.Background(), "1000", []int32{1, 2, 3}, []int32{4, 5, 6}) + err := ts.SetUserOnline(context.Background(), "1000", nil, []int32{1, 2, 3}) + + t.Log(err) + +} + +/* + +local function tableToString(tbl, separator) + local result = {} + for _, v in ipairs(tbl) do + table.insert(result, tostring(v)) + end + return table.concat(result, separator) +end + +local myTable = {"one", "two", "three"} +local result = tableToString(myTable, ":") + +print(result) + +*/ + +func TestRecvOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pubsub := ts.rdb.Subscribe(ctx, "user_online") + + // 等待订阅确认 + _, err := pubsub.Receive(ctx) + if err != nil { + log.Fatalf("Could not subscribe: %v", err) + } + + // 创建一个通道来接收消息 + ch := pubsub.Channel() + + // 处理接收到的消息 + for msg := range ch { + fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) + } +} diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index 3de01563bf..c05cd38957 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -131,6 +131,15 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } +/* + + */ + +type RedisUserOnline struct { + // 平台id, 平台更新时间 + PlatformIDs map[int32]int64 +} + // GetUserStatus get user status. func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) From 2b2a75f19e47daee990d329757ab0a3d3e84fe2f Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 28 Jun 2024 15:29:03 +0800 Subject: [PATCH 066/147] implement minio expire delete. --- config/openim-crontask.yml | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/rpc/third/s3.go | 12 ++++++++++- internal/tools/cron_task.go | 25 +++++++++++++++++++++-- pkg/common/config/config.go | 6 ++++-- pkg/common/storage/controller/s3.go | 9 ++++++-- pkg/common/storage/database/mgo/object.go | 7 +++++++ pkg/common/storage/database/object.go | 3 +++ pkg/rpcclient/third.go | 4 ++++ 10 files changed, 63 insertions(+), 10 deletions(-) diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 9bbccfd252..d2154d2638 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,2 +1,3 @@ chatRecordsClearTime: "0 2 * * *" retainChatRecords: 365 +fileExpireTime: 90 diff --git a/go.mod b/go.mod index 2614e0f322..a1a51b4fa2 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/protocol v0.0.69-alpha.22 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index fe4f0c390a..e67b30206a 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= +github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 4cb1b81d06..19a519d2e9 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,11 +19,12 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/third" @@ -283,6 +284,15 @@ func (t *thirdServer) apiAddress(prefix, name string) string { return prefix + name } +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + expireTime := time.UnixMilli(req.ExpireTime) + err := t.s3dataBase.DeleteByExpires(ctx, expireTime) + if err != nil { + return nil, err + } + return &third.DeleteOutdatedDataResp{}, nil +} + type FormDataMate struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694b..28963dc631 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -17,15 +17,17 @@ package tools import ( "context" "fmt" + "os" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "os" - "time" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -69,6 +71,25 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { return errs.Wrap(err) } + deleteFunc := func() { + now := time.Now() + deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return + } + thirdClient := third.NewThirdClient(tConn) + if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + return + } + log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(string(config.CronTask.FileExpireTime), deleteFunc); err != nil { + return errs.Wrap(err) + } log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) crontab.Start() <-ctx.Done() diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f5..7e56499877 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,14 +15,15 @@ package config import ( + "strings" + "time" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "strings" - "time" ) type CacheConfig struct { @@ -107,6 +108,7 @@ type API struct { type CronTask struct { ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b0ad612038..5d83e26771 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -16,11 +16,12 @@ package controller import ( "context" + "path/filepath" + "time" + redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "path/filepath" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/s3" @@ -38,6 +39,7 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) + DeleteByExpires(ctx context.Context, duration time.Time) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -111,3 +113,6 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } +func (s *s3Database) DeleteByExpires(ctx context.Context, duration time.Time) error { + return s.db.DeleteByExpires(ctx, duration) +} diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index df4d10ec4f..689c869d5f 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -16,6 +16,8 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -68,3 +70,8 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } +func (o *S3Mongo) DeleteByExpires(ctx context.Context, duration time.Time) error { + return mongoutil.DeleteMany(ctx, o.coll, bson.M{ + "create_time": bson.M{"$lt": duration}, + }) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 554f71f35d..3752993d85 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -16,6 +16,8 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -23,4 +25,5 @@ type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error + DeleteByExpires(ctx context.Context, duration time.Time) error } diff --git a/pkg/rpcclient/third.go b/pkg/rpcclient/third.go index 4c71dff6ad..7cdc60d52f 100644 --- a/pkg/rpcclient/third.go +++ b/pkg/rpcclient/third.go @@ -41,3 +41,7 @@ func NewThird(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, grafanaUrl } return &Third{discov: discov, Client: client, conn: conn, GrafanaUrl: grafanaUrl} } +func (t *Third) DeleteOutdatedData(ctx context.Context, expires int64) error { + _, err := t.Client.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expires}) + return err +} From 8f86049599afe62a85a612ac85dee72dcad583d8 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:18:21 +0800 Subject: [PATCH 067/147] user online --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/init.go | 7 +- internal/msggateway/n_ws_server.go | 42 +-- internal/msggateway/online.go | 106 ++++++++ internal/msggateway/user_map.go | 239 +++++++++--------- internal/msggateway/user_map2.go | 208 ++++++++++++--- internal/rpc/user/online.go | 122 +++++++++ internal/rpc/user/user.go | 73 +----- internal/tools/cron_task.go | 6 + pkg/common/config/config.go | 1 + pkg/common/storage/cache/cachekey/online.go | 13 + pkg/common/storage/cache/cachekey/user.go | 5 - pkg/common/storage/cache/online.go | 8 + pkg/common/storage/cache/redis/online.go | 33 ++- .../storage/cache/redis/seq_user_test.go | 53 ++-- pkg/common/storage/cache/redis/user.go | 183 +++++++------- pkg/common/storage/cache/user.go | 5 +- pkg/common/storage/controller/user.go | 15 -- 19 files changed, 740 insertions(+), 385 deletions(-) create mode 100644 internal/msggateway/online.go create mode 100644 internal/rpc/user/online.go create mode 100644 pkg/common/storage/cache/cachekey/online.go create mode 100644 pkg/common/storage/cache/online.go diff --git a/go.mod b/go.mod index 2614e0f322..e7ed446f47 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/protocol v0.0.69-alpha.20 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index fe4f0c390a..adde3aa82b 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.20 h1:skZu82sqoMhiQVEZgrRsjcfI3Grp1IpThx1LJPqETWs= +github.com/openimsdk/protocol v0.0.69-alpha.20/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index f4d8b0381a..815ec8ca60 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -42,16 +42,15 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } - longServer, err := NewWsServer( + longServer := NewWsServer( conf, WithPort(wsPort), WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)), WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second), WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - if err != nil { - return err - } + + go longServer.ChangeOnlineStatus(4) hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 17e550fefc..ff93ce6bc3 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -60,7 +60,7 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients *UserMap + clients UMap clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -90,18 +90,18 @@ func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, ws.disCov = disCov } -func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { - err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) - if err != nil { - log.ZWarn(ctx, "SetUserStatus err", err) - } - switch status { - case constant.Online: - ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) - case constant.Offline: - ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) - } -} +//func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { +// err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) +// if err != nil { +// log.ZWarn(ctx, "SetUserStatus err", err) +// } +// switch status { +// case constant.Online: +// ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) +// case constant.Offline: +// ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) +// } +//} func (ws *WsServer) UnRegister(c *Client) { ws.unregisterChan <- c @@ -119,7 +119,7 @@ func (ws *WsServer) GetUserPlatformCons(userID string, platform int) ([]*Client, return ws.clients.Get(userID, platform) } -func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { +func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { var config configs for _, o := range opts { o(&config) @@ -144,7 +144,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), - }, nil + } } func (ws *WsServer) Run(done chan error) error { @@ -278,11 +278,11 @@ func (ws *WsServer) registerClient(client *Client) { }() } - wg.Add(1) - go func() { - defer wg.Done() - ws.SetUserOnlineStatus(client.ctx, client, constant.Online) - }() + //wg.Add(1) + //go func() { + // defer wg.Done() + // ws.SetUserOnlineStatus(client.ctx, client, constant.Online) + //}() wg.Wait() @@ -351,7 +351,7 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) + //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go new file mode 100644 index 0000000000..611ecbd79b --- /dev/null +++ b/internal/msggateway/online.go @@ -0,0 +1,106 @@ +package msggateway + +import ( + "context" + "crypto/md5" + "encoding/binary" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" + "math/rand" + "strconv" + "time" +) + +func (ws *WsServer) ChangeOnlineStatus(concurrent int) { + if concurrent < 1 { + concurrent = 1 + } + scanTicker := time.NewTicker(time.Minute * 3) + + requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) + changeStatus := make([][]UserState, concurrent) + + for i := 0; i < concurrent; i++ { + requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) + changeStatus[i] = make([]UserState, 100) + } + + mergeTicker := time.NewTicker(time.Second) + + local2pb := func(u UserState) *pbuser.UserOnlineStatus { + return &pbuser.UserOnlineStatus{ + UserID: u.UserID, + Online: u.Online, + Offline: u.Offline, + } + } + + rNum := rand.Uint64() + pushUserState := func(us ...UserState) { + for _, u := range us { + sum := md5.Sum([]byte(u.UserID)) + i := (binary.BigEndian.Uint64(sum[:]) + rNum) % uint64(concurrent) + changeStatus[i] = append(changeStatus[i], u) + status := changeStatus[i] + if len(status) == cap(status) { + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[0:] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + } + + pushAllUserState := func() { + for i, status := range changeStatus { + if len(status) == 0 { + continue + } + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[0:] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + + opIdCtx := mcontext.SetOperationID(context.Background(), "r"+strconv.FormatUint(rNum, 10)) + doRequest := func(req *pbuser.SetUserOnlineStatusReq) { + ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) + defer cancel() + if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { + log.ZError(ctx, "update user online status", err) + } + } + + for i := 0; i < concurrent; i++ { + go func(ch <-chan *pbuser.SetUserOnlineStatusReq) { + for req := range ch { + doRequest(req) + } + }(requestChs[i]) + } + + for { + select { + case <-mergeTicker.C: + pushAllUserState() + case now := <-scanTicker.C: + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + case state := <-ws.clients.UserState(): + pushUserState(state) + } + } +} diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index f8bf69f9ad..dcab066f84 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -14,122 +14,123 @@ package msggateway -import ( - "context" - "sync" - - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" -) - -type UserMap struct { - m sync.Map -} - -func newUserMap() *UserMap { - return &UserMap{} -} - -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.([]*Client), ok - } - return nil, ok -} - -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.([]*Client) { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, userExisted, true - } - return clients, userExisted, false - } - return nil, userExisted, false -} - -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.([]*Client) - oldClients = append(oldClients, v) - u.m.Store(key, oldClients) - } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) - - var clients []*Client - clients = append(clients, v) - u.m.Store(key, clients) - } -} - -func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { - // Attempt to load the clients associated with the key. - allClients, existed := u.m.Load(key) - if !existed { - // Return false immediately if the key does not exist. - return false - } - - // Convert allClients to a slice of *Client. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - // Keep clients that do not match the connRemoteAddr. - if client.ctx.GetRemoteAddr() != connRemoteAddr { - remainingClients = append(remainingClients, client) - } - } - - // If no clients remain after filtering, delete the key from the map. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true - } - - // Otherwise, update the key with the remaining clients. - u.m.Store(key, remainingClients) - return false -} - -func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. - return false - } - - // Filter out clients that are in the deleteMap. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) - } - } - - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true - } - - u.m.Store(key, remainingClients) - return false -} - -func (u *UserMap) DeleteAll(key string) { - u.m.Delete(key) -} +// +//import ( +// "context" +// "sync" +// +// "github.com/openimsdk/tools/log" +// "github.com/openimsdk/tools/utils/datautil" +//) +// +//type UserMap struct { +// m sync.Map +//} +// +//func newUserMap() UMap { +// return &UserMap{} +//} +// +//func (u *UserMap) GetAll(key string) ([]*Client, bool) { +// allClients, ok := u.m.Load(key) +// if ok { +// return allClients.([]*Client), ok +// } +// return nil, ok +//} +// +//func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { +// allClients, userExisted := u.m.Load(key) +// if userExisted { +// var clients []*Client +// for _, client := range allClients.([]*Client) { +// if client.PlatformID == platformID { +// clients = append(clients, client) +// } +// } +// if len(clients) > 0 { +// return clients, userExisted, true +// } +// return clients, userExisted, false +// } +// return nil, userExisted, false +//} +// +//// Set adds a client to the map. +//func (u *UserMap) Set(key string, v *Client) { +// allClients, existed := u.m.Load(key) +// if existed { +// log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) +// oldClients := allClients.([]*Client) +// oldClients = append(oldClients, v) +// u.m.Store(key, oldClients) +// } else { +// log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) +// +// var clients []*Client +// clients = append(clients, v) +// u.m.Store(key, clients) +// } +//} +// +//func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { +// // Attempt to load the clients associated with the key. +// allClients, existed := u.m.Load(key) +// if !existed { +// // Return false immediately if the key does not exist. +// return false +// } +// +// // Convert allClients to a slice of *Client. +// oldClients := allClients.([]*Client) +// var remainingClients []*Client +// for _, client := range oldClients { +// // Keep clients that do not match the connRemoteAddr. +// if client.ctx.GetRemoteAddr() != connRemoteAddr { +// remainingClients = append(remainingClients, client) +// } +// } +// +// // If no clients remain after filtering, delete the key from the map. +// if len(remainingClients) == 0 { +// u.m.Delete(key) +// return true +// } +// +// // Otherwise, update the key with the remaining clients. +// u.m.Store(key, remainingClients) +// return false +//} +// +//func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { +// m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { +// return c.ctx.GetRemoteAddr(), struct{}{} +// }) +// allClients, existed := u.m.Load(key) +// if !existed { +// // If the key doesn't exist, return false. +// return false +// } +// +// // Filter out clients that are in the deleteMap. +// oldClients := allClients.([]*Client) +// var remainingClients []*Client +// for _, client := range oldClients { +// if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { +// remainingClients = append(remainingClients, client) +// } +// } +// +// // Update or delete the key based on the remaining clients. +// if len(remainingClients) == 0 { +// u.m.Delete(key) +// return true +// } +// +// u.m.Store(key, remainingClients) +// return false +//} +// +//func (u *UserMap) DeleteAll(key string) { +// u.m.Delete(key) +//} diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 1c9a91c6a6..b6ed403739 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,32 +1,180 @@ package msggateway -/* - -sorted set - -userID: 10000 - -USER_ONLINE:10000 - - - - - -platformID: 1 - - - - - -key score - -E1 123456789 -O1 234567895 - - - - - - - -*/ +import ( + "sync" + "time" +) + +type UMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + Delete(userID string, connRemoteAddr string) (isDeleteUser bool) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline, nowtime time.Time) []UserState +} + +var _ UMap = (*UserMap2)(nil) + +type UserPlatform struct { + Time time.Time + Clients map[string]*Client +} + +func (u *UserPlatform) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) + } + return platformIDs +} + +func newUserMap() UMap { + return &UserMap2{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), + } +} + +type UserMap2 struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState +} + +func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() + return true + default: + return false + } +} + +func (u *UserMap2) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false + } + clients := make([]*Client, 0, len(result.Clients)) + for _, client := range result.Clients { + clients = append(clients, client) + } + return clients, true +} + +func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false + } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + return clients, true, len(clients) > 0 +} + +func (u *UserMap2) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients[client.ctx.GetRemoteAddr()] = client + } else { + result = &UserPlatform{ + Clients: map[string]*Client{ + client.ctx.GetRemoteAddr(): client, + }, + } + } + u.push(client.UserID, result, nil) +} + +func (u *UserMap2) Delete(userID string, connRemoteAddr string) (isDeleteUser bool) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + client, ok := result.Clients[connRemoteAddr] + if !ok { + return false + } + delete(result.Clients, connRemoteAddr) + defer u.push(userID, result, []int32{int32(client.PlatformID)}) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} + +func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { + return false + } + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + for _, client := range clients { + offline = append(offline, int32(client.PlatformID)) + delete(result.Clients, client.ctx.GetRemoteAddr()) + } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} + +func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { + u.lock.RLock() + defer u.lock.RUnlock() + if len(u.data) == 0 { + return nil + } + result := make([]UserState, 0, len(u.data)) + for userID, p := range u.data { + if len(result) == cap(result) { + break + } + if p.Time.Before(deadline) { + continue + } + p.Time = nowtime + online := make([]int32, 0, len(p.Clients)) + for _, client := range p.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) + } + return result +} + +func (u *UserMap2) UserState() <-chan UserState { + return u.ch +} + +type UserState struct { + UserID string + Online []int32 + Offline []int32 +} diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go new file mode 100644 index 0000000000..e853ceae23 --- /dev/null +++ b/internal/rpc/user/online.go @@ -0,0 +1,122 @@ +package user + +import ( + "context" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" + pbuser "github.com/openimsdk/protocol/user" +) + +func (s *userServer) getUserOnlineStatus(ctx context.Context, userID string) (*pbuser.OnlineStatus, error) { + platformIDs, err := s.online.GetOnline(ctx, userID) + if err != nil { + return nil, err + } + status := pbuser.OnlineStatus{ + UserID: userID, + PlatformIDs: platformIDs, + } + if len(platformIDs) > 0 { + status.Status = constant.Online + } else { + status.Status = constant.Offline + } + return &status, nil +} + +func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) ([]*pbuser.OnlineStatus, error) { + res := make([]*pbuser.OnlineStatus, 0, len(userIDs)) + for _, userID := range userIDs { + status, err := s.getUserOnlineStatus(ctx, userID) + if err != nil { + return nil, err + } + res = append(res, status) + } + return res, nil +} + +// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. +func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { + if req.Genre == constant.SubscriberUser { + err := s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + var status []*pbuser.OnlineStatus + status, err = s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil + } else if req.Genre == constant.Unsubscribe { + err := s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + } + return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil +} + +// GetUserStatus Get the online status of the user. +func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (*pbuser.GetUserStatusResp, error) { + res, err := s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.GetUserStatusResp{StatusList: res}, nil +} + +// SetUserStatus Synchronize user's online status. +func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (*pbuser.SetUserStatusResp, error) { + var ( + online []int32 + offline []int32 + ) + switch req.Status { + case constant.Online: + online = []int32{req.PlatformID} + case constant.Offline: + online = []int32{req.PlatformID} + } + if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { + return nil, err + } + list, err := s.db.GetSubscribedList(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range list { + tips := &sdkws.UserStatusChangeTips{ + FromUserID: req.UserID, + ToUserID: userID, + Status: req.Status, + PlatformID: req.PlatformID, + } + s.userNotificationSender.UserStatusChangeNotification(ctx, tips) + } + + return &pbuser.SetUserStatusResp{}, nil +} + +// GetSubscribeUsersStatus Get the online status of subscribers. +func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) + if err != nil { + return nil, err + } + onlineStatusList, err := s.getUsersOnlineStatus(ctx, userList) + if err != nil { + return nil, err + } + return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil +} + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + for _, status := range req.Status { + if err := s.online.SetUserOnline(ctx, status.UserID, status.Online, status.Offline); err != nil { + return nil, err + } + } + return &pbuser.SetUserOnlineStatusResp{}, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7b..0b96077ec3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -19,6 +19,7 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -50,6 +51,7 @@ import ( ) type userServer struct { + online cache.OnlineCache db controller.UserDatabase friendNotificationSender *friend.FriendNotificationSender userNotificationSender *UserNotificationSender @@ -98,6 +100,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ + online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, friendRpcClient: &friendRpcClient, @@ -329,76 +332,6 @@ func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDR return &pbuser.GetAllUserIDResp{Total: int32(total), UserIDs: userIDs}, nil } -// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. -func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (resp *pbuser.SubscribeOrCancelUsersStatusResp, err error) { - if req.Genre == constant.SubscriberUser { - err = s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err = s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } - return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil -} - -// GetUserStatus Get the online status of the user. -func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { - onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.GetUserStatusResp{StatusList: onlineStatusList}, nil -} - -// SetUserStatus Synchronize user's online status. -func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { - err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) - if err != nil { - return nil, err - } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - - return &pbuser.SetUserStatusResp{}, nil -} - -// GetSubscribeUsersStatus Get the online status of subscribers. -func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.db.GetUserStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil -} - // ProcessUserCommandAdd user general function add. func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.ProcessUserCommandAddReq) (*pbuser.ProcessUserCommandAddResp, error) { err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694b..50f328b555 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -65,10 +65,16 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + } + delFile := func() { + } if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { return errs.Wrap(err) } + if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, delFile); err != nil { + return errs.Wrap(err) + } log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) crontab.Start() <-ctx.Done() diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f5..a4e6135606 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -107,6 +107,7 @@ type API struct { type CronTask struct { ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` RetainChatRecords int `mapstructure:"retainChatRecords"` + FileTime int } type OfflinePushConfig struct { diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go new file mode 100644 index 0000000000..164e5f2f46 --- /dev/null +++ b/pkg/common/storage/cache/cachekey/online.go @@ -0,0 +1,13 @@ +package cachekey + +import "time" + +const ( + OnlineKey = "ONLINE:" + OnlineChannel = "online_change" + OnlineExpire = time.Hour / 2 +) + +func GetOnlineKey(userID string) string { + return OnlineKey + userID +} diff --git a/pkg/common/storage/cache/cachekey/user.go b/pkg/common/storage/cache/cachekey/user.go index 7d06d4f751..473ca1b12f 100644 --- a/pkg/common/storage/cache/cachekey/user.go +++ b/pkg/common/storage/cache/cachekey/user.go @@ -17,7 +17,6 @@ package cachekey const ( UserInfoKey = "USER_INFO:" UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:" - olineStatusKey = "ONLINE_STATUS:" ) func GetUserInfoKey(userID string) string { @@ -27,7 +26,3 @@ func GetUserInfoKey(userID string) string { func GetUserGlobalRecvMsgOptKey(userID string) string { return UserGlobalRecvMsgOptKey + userID } - -func GetOnlineStatusKey(modKey string) string { - return olineStatusKey + modKey -} diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go new file mode 100644 index 0000000000..7669c8a118 --- /dev/null +++ b/pkg/common/storage/cache/online.go @@ -0,0 +1,8 @@ +package cache + +import "context" + +type OnlineCache interface { + GetOnline(ctx context.Context, userID string) ([]int32, error) + SetUserOnline(ctx context.Context, userID string, online, offline []int32) error +} diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index 138f9a5738..dc6a5f775b 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -2,10 +2,22 @@ package redis import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" + "strconv" "time" ) +func NewUserOnline(rdb redis.UniversalClient) cache.OnlineCache { + return &userOnline{ + rdb: rdb, + expire: cachekey.OnlineExpire, + channelName: cachekey.OnlineChannel, + } +} + type userOnline struct { rdb redis.UniversalClient expire time.Duration @@ -13,7 +25,26 @@ type userOnline struct { } func (s *userOnline) getUserOnlineKey(userID string) string { - return "USER_ONLINE:" + userID + return cachekey.GetOnlineKey(userID) +} + +func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, error) { + members, err := s.rdb.ZRangeByScore(ctx, s.getUserOnlineKey(userID), &redis.ZRangeBy{ + Min: strconv.FormatInt(time.Now().Unix(), 10), + Max: "+inf", + }).Result() + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs := make([]int32, 0, len(members)) + for _, member := range members { + val, err := strconv.Atoi(member) + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs = append(platformIDs, int32(val)) + } + return platformIDs, nil } func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index 04a5d49cb1..b4c852cd0d 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/redis/go-redis/v9" "log" + "strconv" + "sync/atomic" "testing" "time" ) @@ -24,30 +26,39 @@ func newTestOnline() *userOnline { func TestOnline(t *testing.T) { ts := newTestOnline() + var count atomic.Int64 + for i := 0; i < 64; i++ { + go func(userID string) { + var err error + for i := 0; ; i++ { + if i%2 == 0 { + err = ts.SetUserOnline(context.Background(), userID, []int32{5, 6}, []int32{7, 8, 9}) + } else { + err = ts.SetUserOnline(context.Background(), userID, []int32{1, 2, 3}, []int32{4, 5, 6}) + } + if err != nil { + panic(err) + } + count.Add(1) + } + }(strconv.Itoa(10000 + i)) + } - //err := ts.SetUserOnline(context.Background(), "1000", []int32{1, 2, 3}, []int32{4, 5, 6}) - err := ts.SetUserOnline(context.Background(), "1000", nil, []int32{1, 2, 3}) - - t.Log(err) - + ticker := time.NewTicker(time.Second) + for range ticker.C { + t.Log(count.Swap(0)) + } } -/* - -local function tableToString(tbl, separator) - local result = {} - for _, v in ipairs(tbl) do - table.insert(result, tostring(v)) - end - return table.concat(result, separator) -end - -local myTable = {"one", "two", "three"} -local result = tableToString(myTable, ":") - -print(result) - -*/ +func TestGetOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pIDs, err := ts.GetOnline(ctx, "10000") + if err != nil { + panic(err) + } + t.Log(pIDs) +} func TestRecvOnline(t *testing.T) { ts := newTestOnline() diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index c05cd38957..b846bf3981 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -17,7 +17,6 @@ package redis import ( "context" "encoding/json" - "errors" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -29,8 +28,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "hash/crc32" - "strconv" "time" ) @@ -61,9 +58,9 @@ func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, } } -func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { - return cachekey.GetOnlineStatusKey(modKey) -} +//func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { +// return cachekey.GetOnlineStatusKey(modKey) +//} func (u *UserCacheRedis) CloneUserCache() cache.UserCache { return &UserCacheRedis{ @@ -141,95 +138,95 @@ type RedisUserOnline struct { } // GetUserStatus get user status. -func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) - for _, userID := range userIDs { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - var onlineStatus user.OnlineStatus - key := u.getOnlineStatusKey(modKey) - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - // key or field does not exist - userStatus = append(userStatus, &user.OnlineStatus{ - UserID: userID, - Status: constant.Offline, - PlatformIDs: nil, - }) - - continue - } else { - return nil, errs.Wrap(err) - } - } - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return nil, errs.Wrap(err) - } - onlineStatus.UserID = userID - onlineStatus.Status = constant.Online - userStatus = append(userStatus, &onlineStatus) - } - - return userStatus, nil -} +//func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { +// userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) +// for _, userID := range userIDs { +// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) +// modKey := strconv.Itoa(int(UserIDNum % statusMod)) +// var onlineStatus user.OnlineStatus +// key := u.getOnlineStatusKey(modKey) +// result, err := u.rdb.HGet(ctx, key, userID).Result() +// if err != nil { +// if errors.Is(err, redis.Nil) { +// // key or field does not exist +// userStatus = append(userStatus, &user.OnlineStatus{ +// UserID: userID, +// Status: constant.Offline, +// PlatformIDs: nil, +// }) +// +// continue +// } else { +// return nil, errs.Wrap(err) +// } +// } +// err = json.Unmarshal([]byte(result), &onlineStatus) +// if err != nil { +// return nil, errs.Wrap(err) +// } +// onlineStatus.UserID = userID +// onlineStatus.Status = constant.Online +// userStatus = append(userStatus, &onlineStatus) +// } +// +// return userStatus, nil +//} // SetUserStatus Set the user status and save it in redis. -func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - key := u.getOnlineStatusKey(modKey) - log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) - isNewKey, err := u.rdb.Exists(ctx, key).Result() - if err != nil { - return errs.Wrap(err) - } - if isNewKey == 0 { - if status == constant.Online { - onlineStatus := user.OnlineStatus{ - UserID: userID, - Status: constant.Online, - PlatformIDs: []int32{platformID}, - } - jsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - u.rdb.Expire(ctx, key, userOlineStatusExpireTime) - - return nil - } - } - - isNil := false - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - isNil = true - } else { - return errs.Wrap(err) - } - } - - if status == constant.Offline { - err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) - if err != nil { - return err - } - } else { - err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} +//func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { +// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) +// modKey := strconv.Itoa(int(UserIDNum % statusMod)) +// key := u.getOnlineStatusKey(modKey) +// log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) +// isNewKey, err := u.rdb.Exists(ctx, key).Result() +// if err != nil { +// return errs.Wrap(err) +// } +// if isNewKey == 0 { +// if status == constant.Online { +// onlineStatus := user.OnlineStatus{ +// UserID: userID, +// Status: constant.Online, +// PlatformIDs: []int32{platformID}, +// } +// jsonData, err := json.Marshal(&onlineStatus) +// if err != nil { +// return errs.Wrap(err) +// } +// _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() +// if err != nil { +// return errs.Wrap(err) +// } +// u.rdb.Expire(ctx, key, userOlineStatusExpireTime) +// +// return nil +// } +// } +// +// isNil := false +// result, err := u.rdb.HGet(ctx, key, userID).Result() +// if err != nil { +// if errors.Is(err, redis.Nil) { +// isNil = true +// } else { +// return errs.Wrap(err) +// } +// } +// +// if status == constant.Offline { +// err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) +// if err != nil { +// return err +// } +// } else { +// err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) +// if err != nil { +// return errs.Wrap(err) +// } +// } +// +// return nil +//} func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { if isNil { diff --git a/pkg/common/storage/cache/user.go b/pkg/common/storage/cache/user.go index 5101c0b6ce..69a11635ca 100644 --- a/pkg/common/storage/cache/user.go +++ b/pkg/common/storage/cache/user.go @@ -17,7 +17,6 @@ package cache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/user" ) type UserCache interface { @@ -28,6 +27,6 @@ type UserCache interface { DelUsersInfo(userIDs ...string) UserCache GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error) DelUsersGlobalRecvMsgOpt(userIDs ...string) UserCache - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error + //GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) + //SetUserStatus(ctx context.Context, userID string, status, platformID int32) error } diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 9efe535c01..321eff03cd 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -70,10 +70,6 @@ type UserDatabase interface { GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) // GetSubscribedList Get all subscribed lists GetSubscribedList(ctx context.Context, userID string) ([]string, error) - // GetUserStatus Get the online status of the user - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - // SetUserStatus Set the user status and store the user status in redis - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -246,17 +242,6 @@ func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([] return list, nil } -// GetUserStatus get user status. -func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - onlineStatusList, err := u.cache.GetUserStatus(ctx, userIDs) - return onlineStatusList, err -} - -// SetUserStatus Set the user status and save it in redis. -func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - return u.cache.SetUserStatus(ctx, userID, status, platformID) -} - func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex) } From 3960d28a106c52c7cd402adf2418ed0c25d8b8a1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:22:20 +0800 Subject: [PATCH 068/147] config --- config/discovery.yml | 4 ++-- config/kafka.yml | 2 +- config/minio.yml | 4 ++-- config/mongodb.yml | 2 +- config/redis.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 9cd23c590f..3d96ff9b66 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,13 +1,13 @@ enable: "etcd" etcd: rootDirectory: openim - address: [ 172.16.8.48:12379 ] + address: [ localhost:12379 ] username: '' password: '' zookeeper: schema: openim - address: [ 172.16.8.48:12181 ] + address: [ localhost:12181 ] username: '' password: '' diff --git a/config/kafka.yml b/config/kafka.yml index d9b7ffa3cf..d412e1be06 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -7,7 +7,7 @@ producerAck: "" # Compression type to use (e.g., none, gzip, snappy) compressType: "none" # List of Kafka broker addresses -address: [ 172.16.8.48:19094 ] +address: [ localhost:19094 ] # Kafka topic for Redis integration toRedisTopic: "toRedis" # Kafka topic for MongoDB integration diff --git a/config/minio.yml b/config/minio.yml index d143a1da38..11a9ace354 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -7,9 +7,9 @@ secretAccessKey: "openIM123" # Session token for MinIO authentication (optional) sessionToken: '' # Internal address of the MinIO server -internalAddress: "172.16.8.48:10005" +internalAddress: "localhost:10005" # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://172.16.8.48:10005" +externalAddress: "http://external_ip:10005" # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 53969298b4..98f5694e45 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,7 +1,7 @@ # URI for database connection, leave empty if using address and credential settings directly uri: '' # List of MongoDB server addresses -address: [ 172.16.8.48:37017 ] +address: [ localhost:37017 ] # Name of the database database: openim_v3 # Username for database authentication diff --git a/config/redis.yml b/config/redis.yml index 404d18953f..87abed0e1c 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ 172.16.8.48:16379 ] +address: [ localhost:16379 ] username: '' password: openIM123 clusterMode: false From affa909e175c4f1b800d0e6dec7e73fcf50b351e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:31:53 +0800 Subject: [PATCH 069/147] fix --- internal/msggateway/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 611ecbd79b..a4a1c1f154 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -25,7 +25,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { for i := 0; i < concurrent; i++ { requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) - changeStatus[i] = make([]UserState, 100) + changeStatus[i] = make([]UserState, 0, 100) } mergeTicker := time.NewTicker(time.Second) From 3ef62d4aa5b32df5324cd0dfd02b4344844b3802 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:57:01 +0800 Subject: [PATCH 070/147] fix --- internal/msggateway/online.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index a4a1c1f154..61549083a4 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -49,7 +49,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { req := &pbuser.SetUserOnlineStatusReq{ Status: datautil.Slice(status, local2pb), } - changeStatus[i] = status[0:] + changeStatus[i] = status[:0] select { case requestChs[i] <- req: default: @@ -67,7 +67,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { req := &pbuser.SetUserOnlineStatusReq{ Status: datautil.Slice(status, local2pb), } - changeStatus[i] = status[0:] + changeStatus[i] = status[:0] select { case requestChs[i] <- req: default: @@ -100,6 +100,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case now := <-scanTicker.C: pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): + log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) } } From dac8fba11f325d03c481f9cf0528664b8d3944b2 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 28 Jun 2024 17:59:57 +0800 Subject: [PATCH 071/147] implement minio expire delete logic. --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/third/s3.go | 6 +++++- pkg/common/storage/controller/s3.go | 15 ++++++++++++--- pkg/common/storage/database/mgo/object.go | 4 ++-- pkg/common/storage/database/object.go | 2 +- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index a1a51b4fa2..f8437922ce 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.22 - github.com/openimsdk/tools v0.0.49-alpha.25 + github.com/openimsdk/tools v0.0.49-alpha.30 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index e67b30206a..e05652d991 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= -github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/tools v0.0.49-alpha.30 h1:iT2+1F8cJmlwKEris25YgK0seiJRUear+wTgc1bzcg8= +github.com/openimsdk/tools v0.0.49-alpha.30/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 19a519d2e9..e952175ad9 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -286,10 +286,14 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { expireTime := time.UnixMilli(req.ExpireTime) - err := t.s3dataBase.DeleteByExpires(ctx, expireTime) + models, err := t.s3dataBase.FindByExpires(ctx, expireTime) if err != nil { return nil, err } + for _, model := range models { + t.s3dataBase.DeleteObject(ctx, model.Key) + t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Key) + } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 5d83e26771..b1dce502bf 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -39,7 +39,9 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - DeleteByExpires(ctx context.Context, duration time.Time) error + FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + DeleteObject(ctx context.Context, name string) error + DeleteSpecifiedData(ctx context.Context, engine string, name string) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -113,6 +115,13 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) DeleteByExpires(ctx context.Context, duration time.Time) error { - return s.db.DeleteByExpires(ctx, duration) +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { + return s.db.FindByExpires(ctx, duration) +} + +func (s *s3Database) DeleteObject(ctx context.Context, name string) error { + return s.s3.DeleteObject(ctx, name) +} +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { + return s.db.Delete(ctx, engine, name) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 689c869d5f..a0abf7a7ac 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -70,8 +70,8 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) DeleteByExpires(ctx context.Context, duration time.Time) error { - return mongoutil.DeleteMany(ctx, o.coll, bson.M{ +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ "create_time": bson.M{"$lt": duration}, }) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 3752993d85..44329cbc4a 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -25,5 +25,5 @@ type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error - DeleteByExpires(ctx context.Context, duration time.Time) error + FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) } From 31677806207b394bc86608dc48de3bdd88ae304c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 16:29:13 +0800 Subject: [PATCH 072/147] online cache --- internal/push/push_handler.go | 17 ++- .../storage/cache/redis/seq_user_test.go | 5 +- pkg/localcache/cache.go | 12 +- pkg/localcache/lru/lru.go | 1 + pkg/localcache/lru/lru_expiration.go | 10 ++ pkg/localcache/lru/lru_lazy.go | 22 ++++ pkg/localcache/lru/lru_slot.go | 4 + pkg/localcache/option.go | 14 +- pkg/rpccache/online.go | 121 ++++++++++++++++++ pkg/rpccache/user.go | 15 +++ pkg/rpcclient/user.go | 22 ++++ 11 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 pkg/rpccache/online.go diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 03c299b7ab..029330f1a1 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -45,6 +45,7 @@ type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache msgRpcClient rpcclient.MessageRpcClient @@ -63,16 +64,17 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, - &config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config + consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb) return &consumerHandler, nil } @@ -100,11 +102,13 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { var pushUserIDList []string isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) + pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID}) } else { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) + pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID, pbData.MsgData.SendID}) + } + if err == nil { + err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } if err != nil { log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) @@ -233,7 +237,8 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s } func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { - *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) + //*pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) + *pushToUserIDs, err = c.onlineCache.GetGroupOnline(ctx, groupID) // if err != nil { return err } diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index b4c852cd0d..cfbea004c4 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -3,6 +3,7 @@ package redis import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/redis/go-redis/v9" "log" "strconv" @@ -15,7 +16,7 @@ func newTestOnline() *userOnline { opt := &redis.Options{ Addr: "172.16.8.48:16379", Password: "openIM123", - DB: 1, + DB: 0, } rdb := redis.NewClient(opt) if err := rdb.Ping(context.Background()).Err(); err != nil { @@ -63,7 +64,7 @@ func TestGetOnline(t *testing.T) { func TestRecvOnline(t *testing.T) { ts := newTestOnline() ctx := context.Background() - pubsub := ts.rdb.Subscribe(ctx, "user_online") + pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) // 等待订阅确认 _, err := pubsub.Receive(ctx) diff --git a/pkg/localcache/cache.go b/pkg/localcache/cache.go index 0e040ad389..ba849f8926 100644 --- a/pkg/localcache/cache.go +++ b/pkg/localcache/cache.go @@ -31,6 +31,12 @@ type Cache[V any] interface { Stop() } +func LRUStringHash(key string) uint64 { + h := fnv.New64a() + h.Write(*(*[]byte)(unsafe.Pointer(&key))) + return h.Sum64() +} + func New[V any](opts ...Option) Cache[V] { opt := defaultOption() for _, o := range opts { @@ -49,11 +55,7 @@ func New[V any](opts ...Option) Cache[V] { if opt.localSlotNum == 1 { c.local = createSimpleLRU() } else { - c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, func(key string) uint64 { - h := fnv.New64a() - h.Write(*(*[]byte)(unsafe.Pointer(&key))) - return h.Sum64() - }, createSimpleLRU) + c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, LRUStringHash, createSimpleLRU) } if opt.linkSlotNum > 0 { c.link = link.New(opt.linkSlotNum) diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 64280f238f..2fedffc48b 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,6 +20,7 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + SetHas(key K, value V) bool Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index 970ac083e3..d27e670574 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -89,5 +89,15 @@ func (x *ExpirationLRU[K, V]) Del(key K) bool { return ok } +func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &expirationLruItem[V]{value: value}) + return true + } + return false +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index d6e64aae43..e935c687c4 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,6 +88,28 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } +//func (x *LayLRU[K, V]) Set(key K, value V) { +// x.lock.Lock() +// x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +// x.lock.Unlock() +//} +// +//func (x *LayLRU[K, V]) Has(key K) bool { +// x.lock.Lock() +// defer x.lock.Unlock() +// return x.core.Contains(key) +//} + +func (x *LayLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) + return true + } + return false +} + func (x *LayLRU[K, V]) Del(key K) bool { x.lock.Lock() ok := x.core.Remove(key) diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index d034e94d32..4538ca20e4 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -40,6 +40,10 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) SetHas(key K, value V) bool { + return x.slots[x.getIndex(key)].SetHas(key, value) +} + func (x *slotLRU[K, V]) Del(key K) bool { return x.slots[x.getIndex(key)].Del(key) } diff --git a/pkg/localcache/option.go b/pkg/localcache/option.go index 00bb9d0441..7d91aba6c3 100644 --- a/pkg/localcache/option.go +++ b/pkg/localcache/option.go @@ -30,7 +30,7 @@ func defaultOption() *option { localSuccessTTL: time.Minute, localFailedTTL: time.Second * 5, delFn: make([]func(ctx context.Context, key ...string), 0, 2), - target: emptyTarget{}, + target: EmptyTarget{}, } } @@ -123,14 +123,14 @@ func WithDeleteKeyBefore(fn func(ctx context.Context, key ...string)) Option { } } -type emptyTarget struct{} +type EmptyTarget struct{} -func (e emptyTarget) IncrGetHit() {} +func (e EmptyTarget) IncrGetHit() {} -func (e emptyTarget) IncrGetSuccess() {} +func (e EmptyTarget) IncrGetSuccess() {} -func (e emptyTarget) IncrGetFailed() {} +func (e EmptyTarget) IncrGetFailed() {} -func (e emptyTarget) IncrDelHit() {} +func (e EmptyTarget) IncrDelHit() {} -func (e emptyTarget) IncrDelNotFound() {} +func (e EmptyTarget) IncrDelNotFound() {} diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go new file mode 100644 index 0000000000..79d799bbe8 --- /dev/null +++ b/pkg/rpccache/online.go @@ -0,0 +1,121 @@ +package rpccache + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" + "math/rand" + "strconv" + "strings" + "time" +) + +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient) *OnlineCache { + x := &OnlineCache{ + user: user, + group: group, + local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + }), + } + go func() { + parseUserOnlineStatus := func(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil + } + ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) + for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { + userID, platformIDs, err := parseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + continue + } + x.setUserOnline(userID, platformIDs) + //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { + // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) + //} + } + }() + return x +} + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + local lru.LRU[string, []int32] +} + +func (o *OnlineCache) getUserOnlineKey(userID string) string { + return "" + userID +} + +func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + return o.local.Get(userID, func() ([]int32, error) { + return o.user.GetUserOnlinePlatform(ctx, userID) + }) +} + +func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { + platformIDs, err := o.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return false, err + } + return len(platformIDs) > 0, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { + onlineUserIDs := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + return onlineUserIDs, nil +} + +func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { + userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return nil, err + } + var onlineUserIDs []string + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + return onlineUserIDs, nil +} + +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { + o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) +} diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 25a8eb20d4..6126f5891c 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -110,3 +110,18 @@ func (u *UserLocalCache) GetUsersInfoMap(ctx context.Context, userIDs []string) } return users, nil } + +//func (u *UserLocalCache) GetUserOnlinePlatform(ctx context.Context, userID string) (val []int32, err error) { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform req", "userID", userID) +// defer func() { +// if err == nil { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform return", "value", val) +// } else { +// log.ZError(ctx, "UserLocalCache GetUserOnlinePlatform return", err) +// } +// }() +// return localcache.AnyValue[[]int32](u.local.Get(ctx, cachekey.GetOnlineKey(userID), func(ctx context.Context) (any, error) { +// log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) +// return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) +// })) +//} diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index aab96603eb..cac44fce04 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -193,3 +193,25 @@ func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) }) return err } + +func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { + if len(userIDs) == 0 { + return nil, nil + } + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs}) + if err != nil { + return nil, err + } + return resp.StatusList, nil +} + +func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + resp, err := u.GetUsersOnlinePlatform(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(resp) == 0 { + return nil, nil + } + return resp[0].PlatformIDs, nil +} From d06c323f115fecc9b67fa66a226465251dc86202 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 16:43:19 +0800 Subject: [PATCH 073/147] online cache --- pkg/rpcclient/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index cac44fce04..eabe77b942 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -198,7 +198,7 @@ func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []st if len(userIDs) == 0 { return nil, nil } - resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs}) + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs, UserID: u.imAdminUserID[0]}) if err != nil { return nil, err } From 45b07bc3cc0076d117bb12861dc76881f6b99969 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 17:23:35 +0800 Subject: [PATCH 074/147] online cache --- internal/push/push_handler.go | 15 +++++++++------ pkg/rpccache/online.go | 5 ++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 029330f1a1..33f987528a 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -102,13 +102,11 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { var pushUserIDList []string isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID}) + pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) } else { - pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID, pbData.MsgData.SendID}) - } - if err == nil { - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) + pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) } + err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } if err != nil { log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) @@ -129,7 +127,12 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. -func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) error { +func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { + userIDs, err = c.onlineCache.GetUsersOnline(ctx, userIDs) + if err != nil { + return err + } + log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 79d799bbe8..aeeafcf2bd 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -48,9 +48,10 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { userID, platformIDs, err := parseUserOnlineStatus(message.Payload) if err != nil { - log.ZError(ctx, "redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } + log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload) x.setUserOnline(userID, platformIDs) //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) @@ -95,6 +96,7 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s onlineUserIDs = append(onlineUserIDs, userID) } } + log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) return onlineUserIDs, nil } @@ -113,6 +115,7 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str onlineUserIDs = append(onlineUserIDs, userID) } } + log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs) return onlineUserIDs, nil } From 32c5f65d2fd35f501a0db7a9a322e61fa7ad2800 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 14:32:58 +0800 Subject: [PATCH 075/147] online cache --- .../openim-rpc-conversation/main.go | 4 - cmd/openim-rpc/openim-rpc-group/main.go | 4 - internal/msggateway/n_ws_server.go | 4 +- internal/msggateway/online.go | 2 +- internal/msggateway/user_map.go | 270 ++++++++++-------- internal/msggateway/user_map2.go | 27 +- 6 files changed, 156 insertions(+), 155 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-conversation/main.go b/cmd/openim-rpc/openim-rpc-conversation/main.go index 5b3077ccb2..5b2e66c954 100644 --- a/cmd/openim-rpc/openim-rpc-conversation/main.go +++ b/cmd/openim-rpc/openim-rpc-conversation/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewConversationRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/cmd/openim-rpc/openim-rpc-group/main.go b/cmd/openim-rpc/openim-rpc-group/main.go index 44e5509df1..5badf934e9 100644 --- a/cmd/openim-rpc/openim-rpc-group/main.go +++ b/cmd/openim-rpc/openim-rpc-group/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewGroupRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index ff93ce6bc3..b87f93ab6e 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -140,7 +140,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { unregisterChan: make(chan *Client, 1000), kickHandlerChan: make(chan *kickHandler, 1000), validate: v, - clients: newUserMap(), + clients: newUserMap1(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), @@ -345,7 +345,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.Delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.DeleteClients(client.UserID, []*Client{client}) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 61549083a4..b640cdd807 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -98,7 +98,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case <-mergeTicker.C: pushAllUserState() case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire / 3))...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index dcab066f84..119b07ae56 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -14,123 +14,153 @@ package msggateway -// -//import ( -// "context" -// "sync" -// -// "github.com/openimsdk/tools/log" -// "github.com/openimsdk/tools/utils/datautil" -//) -// -//type UserMap struct { -// m sync.Map -//} -// -//func newUserMap() UMap { -// return &UserMap{} -//} -// -//func (u *UserMap) GetAll(key string) ([]*Client, bool) { -// allClients, ok := u.m.Load(key) -// if ok { -// return allClients.([]*Client), ok -// } -// return nil, ok -//} -// -//func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { -// allClients, userExisted := u.m.Load(key) -// if userExisted { -// var clients []*Client -// for _, client := range allClients.([]*Client) { -// if client.PlatformID == platformID { -// clients = append(clients, client) -// } -// } -// if len(clients) > 0 { -// return clients, userExisted, true -// } -// return clients, userExisted, false -// } -// return nil, userExisted, false -//} -// -//// Set adds a client to the map. -//func (u *UserMap) Set(key string, v *Client) { -// allClients, existed := u.m.Load(key) -// if existed { -// log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) -// oldClients := allClients.([]*Client) -// oldClients = append(oldClients, v) -// u.m.Store(key, oldClients) -// } else { -// log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) -// -// var clients []*Client -// clients = append(clients, v) -// u.m.Store(key, clients) -// } -//} -// -//func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { -// // Attempt to load the clients associated with the key. -// allClients, existed := u.m.Load(key) -// if !existed { -// // Return false immediately if the key does not exist. -// return false -// } -// -// // Convert allClients to a slice of *Client. -// oldClients := allClients.([]*Client) -// var remainingClients []*Client -// for _, client := range oldClients { -// // Keep clients that do not match the connRemoteAddr. -// if client.ctx.GetRemoteAddr() != connRemoteAddr { -// remainingClients = append(remainingClients, client) -// } -// } -// -// // If no clients remain after filtering, delete the key from the map. -// if len(remainingClients) == 0 { -// u.m.Delete(key) -// return true -// } -// -// // Otherwise, update the key with the remaining clients. -// u.m.Store(key, remainingClients) -// return false -//} -// -//func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { -// m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { -// return c.ctx.GetRemoteAddr(), struct{}{} -// }) -// allClients, existed := u.m.Load(key) -// if !existed { -// // If the key doesn't exist, return false. -// return false -// } -// -// // Filter out clients that are in the deleteMap. -// oldClients := allClients.([]*Client) -// var remainingClients []*Client -// for _, client := range oldClients { -// if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { -// remainingClients = append(remainingClients, client) -// } -// } -// -// // Update or delete the key based on the remaining clients. -// if len(remainingClients) == 0 { -// u.m.Delete(key) -// return true -// } -// -// u.m.Store(key, remainingClients) -// return false -//} -// -//func (u *UserMap) DeleteAll(key string) { -// u.m.Delete(key) -//} +import ( + "context" + "sync" + "time" + + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" +) + +func newUserMap1() UMap { + return &UserMap{ + ch: make(chan UserState, 1024), + } +} + +type UserPlatform1 struct { + Time time.Time + Clients []*Client +} + +func (u *UserPlatform1) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) + } + return platformIDs +} + +type UserMap struct { + m sync.Map + ch chan UserState +} + +func (u *UserMap) UserState() <-chan UserState { + return u.ch +} + +func (u *UserMap) GetAllUserStatus(deadline time.Time) []UserState { + var result []UserState + u.m.Range(func(key, value any) bool { + client := value.(*UserPlatform1) + if client.Time.Before(deadline) { + return true + } + client.Time = time.Now() + us := UserState{ + UserID: key.(string), + Online: make([]int32, 0, len(client.Clients)), + } + for _, c := range client.Clients { + us.Online = append(us.Online, int32(c.PlatformID)) + } + return true + }) + return result +} + +func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() + return true + default: + return false + } +} + +func (u *UserMap) GetAll(key string) ([]*Client, bool) { + allClients, ok := u.m.Load(key) + if ok { + return allClients.(*UserPlatform1).Clients, ok + } + return nil, ok +} + +func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { + allClients, userExisted := u.m.Load(key) + if userExisted { + var clients []*Client + for _, client := range allClients.(*UserPlatform1).Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + if len(clients) > 0 { + return clients, true, true + } + return clients, true, false + } + return nil, false, false +} + +// Set adds a client to the map. +func (u *UserMap) Set(key string, v *Client) { + allClients, existed := u.m.Load(key) + if existed { + log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) + oldClients := allClients.(*UserPlatform1) + oldClients.Time = time.Now() + oldClients.Clients = append(oldClients.Clients, v) + u.push(key, oldClients, nil) + } else { + log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) + cli := &UserPlatform1{ + Time: time.Now(), + Clients: []*Client{v}, + } + u.m.Store(key, cli) + u.push(key, cli, nil) + } + +} + +func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { + m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { + return c.ctx.GetRemoteAddr(), struct{}{} + }) + allClients, existed := u.m.Load(key) + if !existed { + // If the key doesn't exist, return false. + return false + } + + // Filter out clients that are in the deleteMap. + oldClients := allClients.(*UserPlatform1) + var ( + remainingClients []*Client + offline []int32 + ) + for _, client := range oldClients.Clients { + if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { + remainingClients = append(remainingClients, client) + } else { + offline = append(offline, int32(client.PlatformID)) + } + } + + oldClients.Clients = remainingClients + defer u.push(key, oldClients, offline) + // Update or delete the key based on the remaining clients. + if len(remainingClients) == 0 { + u.m.Delete(key) + return true + } + + return false +} diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index b6ed403739..8a064ab83e 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -9,10 +9,9 @@ type UMap interface { GetAll(userID string) ([]*Client, bool) Get(userID string, platformID int) ([]*Client, bool, bool) Set(userID string, v *Client) - Delete(userID string, connRemoteAddr string) (isDeleteUser bool) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState - GetAllUserStatus(deadline, nowtime time.Time) []UserState + GetAllUserStatus(deadline time.Time) []UserState } var _ UMap = (*UserMap2)(nil) @@ -102,26 +101,6 @@ func (u *UserMap2) Set(userID string, client *Client) { u.push(client.UserID, result, nil) } -func (u *UserMap2) Delete(userID string, connRemoteAddr string) (isDeleteUser bool) { - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if !ok { - return false - } - client, ok := result.Clients[connRemoteAddr] - if !ok { - return false - } - delete(result.Clients, connRemoteAddr) - defer u.push(userID, result, []int32{int32(client.PlatformID)}) - if len(result.Clients) > 0 { - return false - } - delete(u.data, userID) - return true -} - func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { if len(clients) == 0 { return false @@ -145,7 +124,7 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return true } -func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { +func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { u.lock.RLock() defer u.lock.RUnlock() if len(u.data) == 0 { @@ -159,7 +138,7 @@ func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { if p.Time.Before(deadline) { continue } - p.Time = nowtime + p.Time = time.Now() online := make([]int32, 0, len(p.Clients)) for _, client := range p.Clients { online = append(online, int32(client.PlatformID)) From 9388cb61e8661b48726a6deb2e68de248438e6f8 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:25:43 +0800 Subject: [PATCH 076/147] online cache --- internal/msggateway/n_ws_server.go | 2 +- internal/msggateway/online.go | 2 +- internal/msggateway/user_map.go | 4 +- internal/msggateway/user_map2.go | 66 ++++++++++++++---------------- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index b87f93ab6e..58408e1efb 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -140,7 +140,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { unregisterChan: make(chan *Client, 1000), kickHandlerChan: make(chan *kickHandler, 1000), validate: v, - clients: newUserMap1(), + clients: newUserMap(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index b640cdd807..61549083a4 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -98,7 +98,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case <-mergeTicker.C: pushAllUserState() case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire / 3))...) + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 119b07ae56..e1388d5880 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -54,14 +54,14 @@ func (u *UserMap) UserState() <-chan UserState { return u.ch } -func (u *UserMap) GetAllUserStatus(deadline time.Time) []UserState { +func (u *UserMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { var result []UserState u.m.Range(func(key, value any) bool { client := value.(*UserPlatform1) if client.Time.Before(deadline) { return true } - client.Time = time.Now() + client.Time = nowtime us := UserState{ UserID: key.(string), Online: make([]int32, 0, len(client.Clients)), diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 8a064ab83e..74b49d9d4e 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,6 +1,7 @@ package msggateway import ( + "github.com/openimsdk/tools/utils/datautil" "sync" "time" ) @@ -11,14 +12,14 @@ type UMap interface { Set(userID string, v *Client) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState - GetAllUserStatus(deadline time.Time) []UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState } -var _ UMap = (*UserMap2)(nil) +var _ UMap = (*userMap)(nil) type UserPlatform struct { Time time.Time - Clients map[string]*Client + Clients []*Client } func (u *UserPlatform) PlatformIDs() []int32 { @@ -33,19 +34,19 @@ func (u *UserPlatform) PlatformIDs() []int32 { } func newUserMap() UMap { - return &UserMap2{ + return &userMap{ data: make(map[string]*UserPlatform), ch: make(chan UserState, 10000), } } -type UserMap2 struct { +type userMap struct { lock sync.RWMutex data map[string]*UserPlatform ch chan UserState } -func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int32) bool { +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: userPlatform.Time = time.Now() @@ -55,21 +56,17 @@ func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int } } -func (u *UserMap2) GetAll(userID string) ([]*Client, bool) { +func (u *userMap) GetAll(userID string) ([]*Client, bool) { u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] if !ok { return nil, false } - clients := make([]*Client, 0, len(result.Clients)) - for _, client := range result.Clients { - clients = append(clients, client) - } - return clients, true + return result.Clients, true } -func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -85,23 +82,21 @@ func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { return clients, true, len(clients) > 0 } -func (u *UserMap2) Set(userID string, client *Client) { +func (u *userMap) Set(userID string, client *Client) { u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] if ok { - result.Clients[client.ctx.GetRemoteAddr()] = client + result.Clients = append(result.Clients, client) } else { result = &UserPlatform{ - Clients: map[string]*Client{ - client.ctx.GetRemoteAddr(): client, - }, + Clients: []*Client{client}, } } u.push(client.UserID, result, nil) } -func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { if len(clients) == 0 { return false } @@ -112,9 +107,16 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return false } offline := make([]int32, 0, len(clients)) - for _, client := range clients { - offline = append(offline, int32(client.PlatformID)) - delete(result.Clients, client.ctx.GetRemoteAddr()) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { + continue + } + result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) if len(result.Clients) > 0 { @@ -124,23 +126,17 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return true } -func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { u.lock.RLock() defer u.lock.RUnlock() - if len(u.data) == 0 { - return nil - } result := make([]UserState, 0, len(u.data)) - for userID, p := range u.data { - if len(result) == cap(result) { - break - } - if p.Time.Before(deadline) { + for userID, userPlatform := range u.data { + if userPlatform.Time.Before(deadline) { continue } - p.Time = time.Now() - online := make([]int32, 0, len(p.Clients)) - for _, client := range p.Clients { + userPlatform.Time = time.Now() + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { online = append(online, int32(client.PlatformID)) } result = append(result, UserState{UserID: userID, Online: online}) @@ -148,7 +144,7 @@ func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { return result } -func (u *UserMap2) UserState() <-chan UserState { +func (u *userMap) UserState() <-chan UserState { return u.ch } From 28c8b78b311a52a226d7819c4e826c7e8693bece Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:33:31 +0800 Subject: [PATCH 077/147] online cache --- internal/msggateway/user_map2.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 74b49d9d4e..c913f49ccf 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,6 +1,8 @@ package msggateway import ( + "context" + "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "sync" "time" @@ -57,6 +59,7 @@ func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int3 } func (u *userMap) GetAll(userID string) ([]*Client, bool) { + log.ZInfo(context.Background(), "UserMap GetAll", "userID", userID) u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -67,6 +70,7 @@ func (u *userMap) GetAll(userID string) ([]*Client, bool) { } func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + log.ZInfo(context.Background(), "UserMap Get", "userID", userID, "platformID", platformID) u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -83,6 +87,7 @@ func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { } func (u *userMap) Set(userID string, client *Client) { + log.ZInfo(context.Background(), "UserMap Set", "userID", userID, "client", client.ctx.GetRemoteAddr()) u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] @@ -97,6 +102,7 @@ func (u *userMap) Set(userID string, client *Client) { } func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + log.ZInfo(context.Background(), "UserMap DeleteClients", "userID", userID, "client", len(clients)) if len(clients) == 0 { return false } @@ -134,7 +140,7 @@ func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []User if userPlatform.Time.Before(deadline) { continue } - userPlatform.Time = time.Now() + userPlatform.Time = nowtime online := make([]int32, 0, len(userPlatform.Clients)) for _, client := range userPlatform.Clients { online = append(online, int32(client.PlatformID)) From 6e2659c64a7f91faefb804af8c1084913647c2e5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:46:57 +0800 Subject: [PATCH 078/147] online cache --- internal/msggateway/n_ws_server.go | 2 +- internal/msggateway/user_map.go | 208 ++++++++++++++--------------- internal/msggateway/user_map2.go | 161 ---------------------- 3 files changed, 99 insertions(+), 272 deletions(-) delete mode 100644 internal/msggateway/user_map2.go diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 58408e1efb..1af9f014f2 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -60,7 +60,7 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients UMap + clients UserMap clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index e1388d5880..eb03123a55 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,40 +1,32 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package msggateway import ( - "context" + "github.com/openimsdk/tools/utils/datautil" "sync" "time" - - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" ) -func newUserMap1() UMap { - return &UserMap{ - ch: make(chan UserState, 1024), - } +type UserMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState } -type UserPlatform1 struct { +type UserState struct { + UserID string + Online []int32 + Offline []int32 +} + +type UserPlatform struct { Time time.Time Clients []*Client } -func (u *UserPlatform1) PlatformIDs() []int32 { +func (u *UserPlatform) PlatformIDs() []int32 { if len(u.Clients) == 0 { return nil } @@ -45,36 +37,20 @@ func (u *UserPlatform1) PlatformIDs() []int32 { return platformIDs } -type UserMap struct { - m sync.Map - ch chan UserState -} - -func (u *UserMap) UserState() <-chan UserState { - return u.ch +func newUserMap() UserMap { + return &userMap{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), + } } -func (u *UserMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { - var result []UserState - u.m.Range(func(key, value any) bool { - client := value.(*UserPlatform1) - if client.Time.Before(deadline) { - return true - } - client.Time = nowtime - us := UserState{ - UserID: key.(string), - Online: make([]int32, 0, len(client.Clients)), - } - for _, c := range client.Clients { - us.Online = append(us.Online, int32(c.PlatformID)) - } - return true - }) - return result +type userMap struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState } -func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int32) bool { +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: userPlatform.Time = time.Now() @@ -84,83 +60,95 @@ func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int } } -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.(*UserPlatform1).Clients, ok +func (u *userMap) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false } - return nil, ok + return result.Clients, true } -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.(*UserPlatform1).Clients { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, true, true +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false + } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) } - return clients, true, false } - return nil, false, false + return clients, true, len(clients) > 0 } -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.(*UserPlatform1) - oldClients.Time = time.Now() - oldClients.Clients = append(oldClients.Clients, v) - u.push(key, oldClients, nil) +func (u *userMap) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients = append(result.Clients, client) } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) - cli := &UserPlatform1{ - Time: time.Now(), - Clients: []*Client{v}, + result = &UserPlatform{ + Clients: []*Client{client}, } - u.m.Store(key, cli) - u.push(key, cli, nil) + u.data[userID] = result } - + u.push(client.UserID, result, nil) } -func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { return false } - - // Filter out clients that are in the deleteMap. - oldClients := allClients.(*UserPlatform1) - var ( - remainingClients []*Client - offline []int32 - ) - for _, client := range oldClients.Clients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) - } else { - offline = append(offline, int32(client.PlatformID)) + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { + continue } + result.Clients = append(result.Clients, client) } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} - oldClients.Clients = remainingClients - defer u.push(key, oldClients, offline) - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { + u.lock.RLock() + defer u.lock.RUnlock() + result := make([]UserState, 0, len(u.data)) + for userID, userPlatform := range u.data { + if userPlatform.Time.Before(deadline) { + continue + } + userPlatform.Time = nowtime + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) } + return result +} - return false +func (u *userMap) UserState() <-chan UserState { + return u.ch } diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go deleted file mode 100644 index c913f49ccf..0000000000 --- a/internal/msggateway/user_map2.go +++ /dev/null @@ -1,161 +0,0 @@ -package msggateway - -import ( - "context" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" - "sync" - "time" -) - -type UMap interface { - GetAll(userID string) ([]*Client, bool) - Get(userID string, platformID int) ([]*Client, bool, bool) - Set(userID string, v *Client) - DeleteClients(userID string, clients []*Client) (isDeleteUser bool) - UserState() <-chan UserState - GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState -} - -var _ UMap = (*userMap)(nil) - -type UserPlatform struct { - Time time.Time - Clients []*Client -} - -func (u *UserPlatform) PlatformIDs() []int32 { - if len(u.Clients) == 0 { - return nil - } - platformIDs := make([]int32, 0, len(u.Clients)) - for _, client := range u.Clients { - platformIDs = append(platformIDs, int32(client.PlatformID)) - } - return platformIDs -} - -func newUserMap() UMap { - return &userMap{ - data: make(map[string]*UserPlatform), - ch: make(chan UserState, 10000), - } -} - -type userMap struct { - lock sync.RWMutex - data map[string]*UserPlatform - ch chan UserState -} - -func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { - select { - case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: - userPlatform.Time = time.Now() - return true - default: - return false - } -} - -func (u *userMap) GetAll(userID string) ([]*Client, bool) { - log.ZInfo(context.Background(), "UserMap GetAll", "userID", userID) - u.lock.RLock() - defer u.lock.RUnlock() - result, ok := u.data[userID] - if !ok { - return nil, false - } - return result.Clients, true -} - -func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { - log.ZInfo(context.Background(), "UserMap Get", "userID", userID, "platformID", platformID) - u.lock.RLock() - defer u.lock.RUnlock() - result, ok := u.data[userID] - if !ok { - return nil, false, false - } - var clients []*Client - for _, client := range result.Clients { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - return clients, true, len(clients) > 0 -} - -func (u *userMap) Set(userID string, client *Client) { - log.ZInfo(context.Background(), "UserMap Set", "userID", userID, "client", client.ctx.GetRemoteAddr()) - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if ok { - result.Clients = append(result.Clients, client) - } else { - result = &UserPlatform{ - Clients: []*Client{client}, - } - } - u.push(client.UserID, result, nil) -} - -func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { - log.ZInfo(context.Background(), "UserMap DeleteClients", "userID", userID, "client", len(clients)) - if len(clients) == 0 { - return false - } - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if !ok { - return false - } - offline := make([]int32, 0, len(clients)) - deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { - return client.ctx.GetRemoteAddr() - }) - tmp := result.Clients - result.Clients = result.Clients[:0] - for _, client := range tmp { - if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { - continue - } - result.Clients = append(result.Clients, client) - } - defer u.push(userID, result, offline) - if len(result.Clients) > 0 { - return false - } - delete(u.data, userID) - return true -} - -func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { - u.lock.RLock() - defer u.lock.RUnlock() - result := make([]UserState, 0, len(u.data)) - for userID, userPlatform := range u.data { - if userPlatform.Time.Before(deadline) { - continue - } - userPlatform.Time = nowtime - online := make([]int32, 0, len(userPlatform.Clients)) - for _, client := range userPlatform.Clients { - online = append(online, int32(client.PlatformID)) - } - result = append(result, UserState{UserID: userID, Online: online}) - } - return result -} - -func (u *userMap) UserState() <-chan UserState { - return u.ch -} - -type UserState struct { - UserID string - Online []int32 - Offline []int32 -} From f87ee44459682a1517f01b1552f0e79537cc8784 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:52:57 +0800 Subject: [PATCH 079/147] online cache --- internal/msggateway/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 61549083a4..ec03fc8030 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -100,7 +100,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case now := <-scanTicker.C: pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): - log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) + log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) } } From 1bfaf3e2d911eddc106d3eee58e4262dfc624d57 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:56:20 +0800 Subject: [PATCH 080/147] online cache --- internal/msggateway/user_map.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index eb03123a55..a5a4a2de5f 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -121,6 +121,7 @@ func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { continue } + offline = append(offline, int32(client.PlatformID)) result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) From dcd874979e261bec3115a94ebed00c829d37cab6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:16:11 +0800 Subject: [PATCH 081/147] online cache --- internal/msggateway/user_map.go | 8 ++++---- pkg/rpccache/online.go | 13 +++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index a5a4a2de5f..57fa78087a 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -118,11 +118,11 @@ func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser tmp := result.Clients result.Clients = result.Clients[:0] for _, client := range tmp { - if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { - continue + if _, delCli := deleteAddr[client.ctx.GetRemoteAddr()]; delCli { + offline = append(offline, int32(client.PlatformID)) + } else { + result.Clients = append(result.Clients, client) } - offline = append(offline, int32(client.PlatformID)) - result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) if len(result.Clients) > 0 { diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index aeeafcf2bd..5072984b03 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -51,11 +51,8 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } - log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload) - x.setUserOnline(userID, platformIDs) - //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { - // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) - //} + storageCache := x.setUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) } }() return x @@ -115,10 +112,10 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str onlineUserIDs = append(onlineUserIDs, userID) } } - log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs) + log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) return onlineUserIDs, nil } -func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { - o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { + return o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) } From 14aba3bb8976f20651246fd49a0367e9e401ae41 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:26:24 +0800 Subject: [PATCH 082/147] online cache --- pkg/rpccache/online.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5072984b03..5bacd647dd 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -64,10 +64,6 @@ type OnlineCache struct { local lru.LRU[string, []int32] } -func (o *OnlineCache) getUserOnlineKey(userID string) string { - return "" + userID -} - func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { return o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) @@ -117,5 +113,5 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str } func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { - return o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) + return o.local.SetHas(userID, platformIDs) } From 3df39a8382c16866540456868cbd12b1bc41213a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:43:43 +0800 Subject: [PATCH 083/147] online cache --- internal/msggateway/online.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index ec03fc8030..b50608f93c 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -18,7 +18,9 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { if concurrent < 1 { concurrent = 1 } - scanTicker := time.NewTicker(time.Minute * 3) + const renewalTime = cachekey.OnlineExpire / 3 + //const renewalTime = time.Second * 10 + renewalTicker := time.NewTicker(renewalTime) requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) changeStatus := make([][]UserState, concurrent) @@ -97,8 +99,11 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { select { case <-mergeTicker.C: pushAllUserState() - case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + case now := <-renewalTicker.C: + deadline := now.Add(-cachekey.OnlineExpire / 3) + users := ws.clients.GetAllUserStatus(deadline, now) + log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users)) + pushUserState(users...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) From 330611796c390404e3baa46d240b84b459b4ef81 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 2 Jul 2024 18:10:27 +0800 Subject: [PATCH 084/147] feat: implement scheduled delete outdated object in minio. --- config/openim-crontask.yml | 2 +- go.mod | 2 +- go.sum | 4 +-- internal/rpc/third/s3.go | 34 ++++++++++++++++++++--- internal/tools/cron_task.go | 20 +++++++------ pkg/common/config/config.go | 6 ++-- pkg/common/storage/controller/s3.go | 31 ++++++++++++++++----- pkg/common/storage/controller/third.go | 3 +- pkg/common/storage/database/mgo/object.go | 6 ++++ pkg/common/storage/database/object.go | 1 + 10 files changed, 81 insertions(+), 28 deletions(-) diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index d2154d2638..3839104a44 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,3 +1,3 @@ -chatRecordsClearTime: "0 2 * * *" +cronExecuteTime: "0 2 * * *" retainChatRecords: 365 fileExpireTime: 90 diff --git a/go.mod b/go.mod index f8437922ce..49e5b50a41 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.22 - github.com/openimsdk/tools v0.0.49-alpha.30 + github.com/openimsdk/tools v0.0.49-alpha.39 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index e05652d991..a7dd1d6327 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.30 h1:iT2+1F8cJmlwKEris25YgK0seiJRUear+wTgc1bzcg8= -github.com/openimsdk/tools v0.0.49-alpha.30/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= +github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= +github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index e952175ad9..99c21d5bb2 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -24,6 +24,7 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -287,12 +288,37 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { expireTime := time.UnixMilli(req.ExpireTime) models, err := t.s3dataBase.FindByExpires(ctx, expireTime) - if err != nil { - return nil, err + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 { + thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + if err != nil { + return nil, errs.Wrap(err) + } + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, "minio", needDelObjectKeys...) + + t.s3dataBase.DeleteObject(ctx, key) + } } + for _, model := range models { - t.s3dataBase.DeleteObject(ctx, model.Key) - t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Key) + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + if err != nil { + return nil, errs.Wrap(err) + } } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 28963dc631..dcdcf2f40d 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -41,7 +41,7 @@ type CronTaskConfig struct { } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.RetainChatRecords) + log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } @@ -68,29 +68,31 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearFunc); err != nil { return errs.Wrap(err) } + + tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return err + } + thirdClient := third.NewThirdClient(tConn) + deleteFunc := func() { now := time.Now() deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) - if err != nil { - return - } - thirdClient := third.NewThirdClient(tConn) if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) return } log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(string(config.CronTask.FileExpireTime), deleteFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteFunc); err != nil { return errs.Wrap(err) } - log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) + log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 7e56499877..3d49ff5778 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -106,9 +106,9 @@ type API struct { } type CronTask struct { - ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` - FileExpireTime int `mapstructure:"fileExpireTime"` + CronExecuteTime string `mapstructure:"cronExecuteTime"` + RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b1dce502bf..f32d8803e4 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -19,7 +19,7 @@ import ( "path/filepath" "time" - redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + redisCache "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -42,20 +42,25 @@ type S3Database interface { FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) DeleteObject(ctx context.Context, name string) error DeleteSpecifiedData(ctx context.Context, engine string, name string) error + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + DelS3Key(ctx context.Context, engine string, keys ...string) error + GetImageThumbnailKey(ctx context.Context, name string) (string, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { return &s3Database{ - s3: cont.New(redis2.NewS3Cache(rdb, s3), s3), - cache: redis2.NewObjectCacheRedis(rdb, obj), - db: obj, + s3: cont.New(redisCache.NewS3Cache(rdb, s3), s3), + cache: redisCache.NewObjectCacheRedis(rdb, obj), + s3cache: redisCache.NewS3Cache(rdb, s3), + db: obj, } } type s3Database struct { - s3 *cont.Controller - cache cache.ObjectCache - db database.ObjectInfo + s3 *cont.Controller + cache cache.ObjectCache + s3cache cont.S3Cache + db database.ObjectInfo } func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) { @@ -125,3 +130,15 @@ func (s *s3Database) DeleteObject(ctx context.Context, name string) error { func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { return s.db.Delete(ctx, engine, name) } + +func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return s.db.FindNotDelByS3(ctx, key, duration) +} + +func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { + return s.s3cache.DelS3Key(ctx, engine, keys...) +} + +func (s *s3Database) GetImageThumbnailKey(ctx context.Context, name string) (string, error) { + return s.s3.GetImageThumbnailKey(ctx, name) +} diff --git a/pkg/common/storage/controller/third.go b/pkg/common/storage/controller/third.go index 344501466b..a9c2ae403c 100644 --- a/pkg/common/storage/controller/third.go +++ b/pkg/common/storage/controller/third.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/db/pagination" diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index a0abf7a7ac..3ae5b8ec82 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -75,3 +75,9 @@ func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*mod "create_time": bson.M{"$lt": duration}, }) } +func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{ + "key": key, + "create_time": bson.M{"$gt": duration}, + }) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 44329cbc4a..4046da2f3c 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,4 +26,5 @@ type ObjectInfo interface { Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } From 8d4737cb677bd55e65c53d59d8318e12e74c9170 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 10:30:06 +0800 Subject: [PATCH 085/147] update gomake version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 245214b932..365933de65 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.13 + github.com/openimsdk/gomake v0.0.14-alpha.5 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible diff --git a/go.sum b/go.sum index 664f2366a7..a5a620a144 100644 --- a/go.sum +++ b/go.sum @@ -268,8 +268,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= -github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= +github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= From 0b4c8020748981da84e2590d5fb877d08227969c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 10:45:11 +0800 Subject: [PATCH 086/147] update gomake version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3f765805c3..e082dd64c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.13 +RUN go get github.com/openimsdk/gomake@v0.0.14-alpha.5 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] From 7897044e282496dfbbf0e2e20323db3b70f7edfb Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:03:03 +0800 Subject: [PATCH 087/147] implement FindExpires pagination. --- internal/rpc/third/s3.go | 57 ++++++++++++++--------- pkg/common/storage/common/types.go | 12 +++++ pkg/common/storage/controller/s3.go | 8 ++-- pkg/common/storage/database/mgo/object.go | 7 +-- pkg/common/storage/database/object.go | 3 +- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 99c21d5bb2..c419cfc8c0 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -23,6 +23,8 @@ import ( "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" @@ -286,39 +288,48 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) - models, err := t.s3dataBase.FindByExpires(ctx, expireTime) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) + findPagination := &common.FindPagination{ + PageNumber: 1, + ShowNumber: 1000, } - needDelObjectKeys := make([]string, 0) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) - } - - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - for _, key := range needDelObjectKeys { - count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + for { + total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination) if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { return nil, errs.Wrap(err) } - if int(count) < 1 { - thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 { + thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + if err != nil { + return nil, errs.Wrap(err) + } + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) + t.s3dataBase.DeleteObject(ctx, key) + } + } + for _, model := range models { + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) if err != nil { return nil, errs.Wrap(err) } - t.s3dataBase.DeleteObject(ctx, thumbnailKey) - t.s3dataBase.DelS3Key(ctx, "minio", needDelObjectKeys...) - - t.s3dataBase.DeleteObject(ctx, key) } - } - - for _, model := range models { - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) - if err != nil { - return nil, errs.Wrap(err) + if total < int64(findPagination.ShowNumber) { + break } + findPagination.PageNumber++ } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/pkg/common/storage/common/types.go b/pkg/common/storage/common/types.go index 7591211588..a19de205ed 100644 --- a/pkg/common/storage/common/types.go +++ b/pkg/common/storage/common/types.go @@ -24,3 +24,15 @@ type GroupSimpleUserID struct { Hash uint64 MemberNum uint32 } + +type FindPagination struct { + PageNumber int32 + ShowNumber int32 +} + +func (f *FindPagination) GetPageNumber() int32 { + return f.PageNumber +} +func (f *FindPagination) GetShowNumber() int32 { + return f.ShowNumber +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index f32d8803e4..5926595360 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,6 +24,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -39,7 +40,7 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) DeleteObject(ctx context.Context, name string) error DeleteSpecifiedData(ctx context.Context, engine string, name string) error FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) @@ -120,8 +121,9 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { - return s.db.FindByExpires(ctx, duration) +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + + return s.db.FindByExpires(ctx, duration, pagination) } func (s *s3Database) DeleteObject(ctx context.Context, name string) error { diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 3ae5b8ec82..4242fbb53a 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,6 +22,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -70,10 +71,10 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { - return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ "create_time": bson.M{"$lt": duration}, - }) + }, pagination) } func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { return mongoutil.Count(ctx, o.coll, bson.M{ diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 4046da2f3c..8292006a04 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,12 +19,13 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error - FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } From 686ccae18d89e64eb8dfc92bb03ed6fa7140923a Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:10:33 +0800 Subject: [PATCH 088/147] remove unnesseary incr. --- internal/rpc/third/s3.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index c419cfc8c0..7d6d2a375d 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -329,7 +329,6 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if total < int64(findPagination.ShowNumber) { break } - findPagination.PageNumber++ } return &third.DeleteOutdatedDataResp{}, nil } From a421bd11bb34098f61b549f9fdff9e8358ceb6fd Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:14:38 +0800 Subject: [PATCH 089/147] fix uncorrect args call. --- internal/rpc/third/s3.go | 3 ++- pkg/common/storage/common/types.go | 12 ------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 7d6d2a375d..a1e3d857e4 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -30,6 +30,7 @@ import ( "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -290,7 +291,7 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) - findPagination := &common.FindPagination{ + findPagination := &sdkws.RequestPagination{ PageNumber: 1, ShowNumber: 1000, } diff --git a/pkg/common/storage/common/types.go b/pkg/common/storage/common/types.go index a19de205ed..7591211588 100644 --- a/pkg/common/storage/common/types.go +++ b/pkg/common/storage/common/types.go @@ -24,15 +24,3 @@ type GroupSimpleUserID struct { Hash uint64 MemberNum uint32 } - -type FindPagination struct { - PageNumber int32 - ShowNumber int32 -} - -func (f *FindPagination) GetPageNumber() int32 { - return f.PageNumber -} -func (f *FindPagination) GetShowNumber() int32 { - return f.ShowNumber -} From e791cbde969715d918b1451ea275a2e3b297980b Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 15:06:09 +0800 Subject: [PATCH 090/147] online push --- internal/push/push_handler.go | 45 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 33f987528a..3f3ca11e81 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -28,6 +28,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" pbchat "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -128,16 +129,11 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { - userIDs, err = c.onlineCache.GetUsersOnline(ctx, userIDs) - if err != nil { - return err - } - log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, userIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } @@ -186,6 +182,38 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat return true } +func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { + var ( + onlineUserIDs []string + offlineUserIDs []string + ) + for _, userID := range pushToUserIDs { + online, err := c.onlineCache.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + var result []*msggateway.SingleMsgToUserResults + if len(onlineUserIDs) > 0 { + var err error + result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + if err != nil { + return nil, err + } + } + for _, userID := range offlineUserIDs { + result = append(result, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return result, nil +} + func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { log.ZDebug(ctx, "Get super group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) var pushToUserIDs []string @@ -199,7 +227,7 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } @@ -240,8 +268,7 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s } func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { - //*pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) - *pushToUserIDs, err = c.onlineCache.GetGroupOnline(ctx, groupID) // + *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) if err != nil { return err } From 006766cd14a274ea51da6d6565d88eb59d2ff9db Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 15:21:34 +0800 Subject: [PATCH 091/147] online push --- pkg/rpccache/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5bacd647dd..eac2f43af8 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -21,7 +21,7 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re user: user, group: group, local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { - return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) }), } go func() { From fcda73f4bc0eb3f48b74d5567ff25364d853fb4e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:16:31 +0800 Subject: [PATCH 092/147] online push --- cmd/openim-api/main.go | 1 - go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/n_ws_server.go | 22 +++-- internal/msggateway/subscription.go | 148 ++++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 14 deletions(-) create mode 100644 internal/msggateway/subscription.go diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index 58e540c05f..e29ed2a592 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -25,5 +25,4 @@ func main() { if err := cmd.NewApiCmd().Exec(); err != nil { program.ExitWithError(err) } - } diff --git a/go.mod b/go.mod index e7ed446f47..e4f45e9635 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.20 + github.com/openimsdk/protocol v0.0.69-alpha.24 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index adde3aa82b..b0f495e827 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.20 h1:skZu82sqoMhiQVEZgrRsjcfI3Grp1IpThx1LJPqETWs= -github.com/openimsdk/protocol v0.0.69-alpha.20/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 0581a025b4..62c71d5be7 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -72,6 +72,8 @@ type Client struct { closed atomic.Bool closedErr error token string + //subLock sync.Mutex + //subUserIDs map[string]struct{} } // ResetClient updates the client's state with new connection and context information. diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 1af9f014f2..8a79151fe7 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -54,13 +54,14 @@ type LongConnServer interface { } type WsServer struct { - msgGatewayConfig *Config - port int - wsMaxConnNum int64 - registerChan chan *Client - unregisterChan chan *Client - kickHandlerChan chan *kickHandler - clients UserMap + msgGatewayConfig *Config + port int + wsMaxConnNum int64 + registerChan chan *Client + unregisterChan chan *Client + kickHandlerChan chan *kickHandler + clients UserMap + //subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -141,9 +142,10 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), - Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), - webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), + //subscription: newSubscription(), + Compressor: NewGzipCompressor(), + Encoder: NewGobEncoder(), + webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go new file mode 100644 index 0000000000..d7e6060370 --- /dev/null +++ b/internal/msggateway/subscription.go @@ -0,0 +1,148 @@ +package msggateway + +//import ( +// "context" +// "encoding/json" +// "github.com/openimsdk/protocol/constant" +// "github.com/openimsdk/protocol/sdkws" +// "github.com/openimsdk/tools/log" +// "github.com/openimsdk/tools/utils/datautil" +// "github.com/openimsdk/tools/utils/idutil" +// "sync" +// "time" +//) +// +//type subClient struct { +// clients map[string]*Client +//} +// +//func newSubscription() *Subscription { +// return &Subscription{ +// userIDs: make(map[string]*subClient), +// } +//} +// +//type Subscription struct { +// lock sync.RWMutex +// userIDs map[string]*subClient +//} +// +//func (s *Subscription) GetClient(userID string) []*Client { +// s.lock.RLock() +// defer s.lock.RUnlock() +// cs, ok := s.userIDs[userID] +// if !ok { +// return nil +// } +// clients := make([]*Client, 0, len(cs.clients)) +// for _, client := range cs.clients { +// clients = append(clients, client) +// } +// return clients +//} +// +//func (s *Subscription) DelClient(client *Client) { +// client.subLock.Lock() +// userIDs := datautil.Keys(client.subUserIDs) +// for _, userID := range userIDs { +// delete(client.subUserIDs, userID) +// } +// client.subLock.Unlock() +// if len(userIDs) == 0 { +// return +// } +// addr := client.ctx.GetRemoteAddr() +// s.lock.Lock() +// defer s.lock.Unlock() +// for _, userID := range userIDs { +// sub, ok := s.userIDs[userID] +// if !ok { +// continue +// } +// delete(sub.clients, addr) +// if len(sub.clients) == 0 { +// delete(s.userIDs, userID) +// } +// } +//} +// +//func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { +// if len(addUserIDs)+len(delUserIDs) == 0 { +// return +// } +// var ( +// del = make(map[string]struct{}) +// add = make(map[string]struct{}) +// ) +// client.subLock.Lock() +// for _, userID := range delUserIDs { +// if _, ok := client.subUserIDs[userID]; !ok { +// continue +// } +// del[userID] = struct{}{} +// delete(client.subUserIDs, userID) +// } +// for _, userID := range addUserIDs { +// delete(del, userID) +// if _, ok := client.subUserIDs[userID]; ok { +// continue +// } +// client.subUserIDs[userID] = struct{}{} +// } +// client.subLock.Unlock() +// if len(del)+len(add) == 0 { +// return +// } +// addr := client.ctx.GetRemoteAddr() +// s.lock.Lock() +// defer s.lock.Unlock() +// for userID := range del { +// sub, ok := s.userIDs[userID] +// if !ok { +// continue +// } +// delete(sub.clients, addr) +// if len(sub.clients) == 0 { +// delete(s.userIDs, userID) +// } +// } +// for userID := range add { +// sub, ok := s.userIDs[userID] +// if !ok { +// sub = &subClient{clients: make(map[string]*Client)} +// s.userIDs[userID] = sub +// } +// sub.clients[addr] = client +// } +//} +// +//func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { +// clients := ws.subscription.GetClient(userID) +// if len(clients) == 0 { +// return +// } +// msgContent, err := json.Marshal(platformIDs) +// if err != nil { +// log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) +// return +// } +// now := time.Now().UnixMilli() +// msgID := idutil.GetMsgIDByMD5(userID) +// msg := &sdkws.MsgData{ +// SendID: userID, +// ClientMsgID: msgID, +// ServerMsgID: msgID, +// SenderPlatformID: constant.AdminPlatformID, +// SessionType: constant.NotificationChatType, +// ContentType: constant.UserSubscribeOnlineStatusNotification, +// Content: msgContent, +// SendTime: now, +// CreateTime: now, +// } +// for _, client := range clients { +// msg.RecvID = client.UserID +// if err := client.PushMessage(ctx, msg); err != nil { +// log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) +// } +// } +//} From c5cf078cd68e2640b2fcb8a8e0a903d15667a144 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:29:33 +0800 Subject: [PATCH 093/147] resolving conflicts --- go.sum | 8 ++++---- internal/rpc/third/s3.go | 1 - internal/rpc/user/user.go | 5 +++++ pkg/apistruct/manage.go | 2 +- pkg/common/storage/cache/redis/seq.go | 0 5 files changed, 10 insertions(+), 6 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/seq.go diff --git a/go.sum b/go.sum index a5a620a144..f452bcd871 100644 --- a/go.sum +++ b/go.sum @@ -270,10 +270,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= -github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= +github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= +github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index a1e3d857e4..54cbc30c1a 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -24,7 +24,6 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7b..2fea232b4f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -60,6 +60,11 @@ type userServer struct { webhookClient *webhook.Client } +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.User RedisConfig config.Redis diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index 6ea6a29ed6..f4deb9fb1b 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -15,7 +15,7 @@ package apistruct import ( - sdkws "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/sdkws" ) // SendMsg defines the structure for sending messages with various metadata. diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index e69de29bb2..0000000000 From a9ab9baa2300c3e603ec4c1b11294ca8d9697401 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:36:18 +0800 Subject: [PATCH 094/147] resolving conflicts --- go.sum | 4 ++-- internal/rpc/user/user.go | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/go.sum b/go.sum index f452bcd871..8d2eb40bb5 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= -github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 1da923e956..0b96077ec3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -62,11 +62,6 @@ type userServer struct { webhookClient *webhook.Client } -func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.User RedisConfig config.Redis From 97636c4c7adc9d07871f8eb9ac9a402a043a1d38 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:51:26 +0800 Subject: [PATCH 095/147] test --- pkg/common/storage/cache/redis/seq_user_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index cfbea004c4..e4fd95922a 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -66,16 +66,13 @@ func TestRecvOnline(t *testing.T) { ctx := context.Background() pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) - // 等待订阅确认 _, err := pubsub.Receive(ctx) if err != nil { log.Fatalf("Could not subscribe: %v", err) } - // 创建一个通道来接收消息 ch := pubsub.Channel() - // 处理接收到的消息 for msg := range ch { fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) } From 1336b83142ca27a13181abba684a1fb2fa6e9ad2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:10:26 +0800 Subject: [PATCH 096/147] api prommetrics --- go.mod | 6 +- go.sum | 4 +- internal/api/init.go | 7 +- internal/api/router.go | 20 +- pkg/common/ginprometheus/ginprometheus.go | 855 +++++++++++----------- pkg/common/prommetrics/api.go | 106 +++ pkg/common/prommetrics/api_test.go | 56 ++ pkg/common/prommetrics/gin_api.go | 18 +- pkg/common/prommetrics/prommetrics.go | 17 +- 9 files changed, 636 insertions(+), 453 deletions(-) create mode 100644 pkg/common/prommetrics/api.go create mode 100644 pkg/common/prommetrics/api_test.go diff --git a/go.mod b/go.mod index 285f0d9653..58a9478ac7 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.24 - github.com/openimsdk/tools v0.0.49-alpha.39 + github.com/openimsdk/tools v0.0.49-alpha.41 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -176,3 +176,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/tools => /Users/chao/Desktop/withchao/tools +//) diff --git a/go.sum b/go.sum index 8d2eb40bb5..d741a538aa 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= -github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= +github.com/openimsdk/tools v0.0.49-alpha.41 h1:rLnwW/yYqtuonDB61U8KB/HXj0BMP4chzF0GLgJL+Uo= +github.com/openimsdk/tools v0.0.49-alpha.41/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index 23866c4a07..e6fe46d22d 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,7 +29,6 @@ import ( "time" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" @@ -72,9 +71,9 @@ func Start(ctx context.Context, index int, config *Config) error { netDone <- struct{}{} return } - p := ginprom.NewPrometheus("app", prommetrics.GetGinCusMetrics("Api")) - p.SetListenAddress(fmt.Sprintf(":%d", prometheusPort)) - if err = p.Use(router); err != nil && err != http.ErrServerClosed { + srv := http.NewServeMux() + srv.Handle(prommetrics.ApiPath, prommetrics.ApiHandler()) + if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } diff --git a/internal/api/router.go b/internal/api/router.go index 0f46f26baf..b519c30ff2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,8 +2,10 @@ package api import ( "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "net/http" "strings" + "time" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" @@ -19,6 +21,22 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +func prommetricsGin() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + path := c.FullPath() + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) + if c.Request.Method == http.MethodPost { + if resp := apiresp.GetGinApiResponse(c); resp == nil { + prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") + } else { + prommetrics.APICall(path, resp.ErrCode, resp.ErrMsg) + } + } + } +} + func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) @@ -37,7 +55,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - r.Use(gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) userRouterGroup := r.Group("/user") diff --git a/pkg/common/ginprometheus/ginprometheus.go b/pkg/common/ginprometheus/ginprometheus.go index c2e6bdcca0..64f8a0d8a4 100644 --- a/pkg/common/ginprometheus/ginprometheus.go +++ b/pkg/common/ginprometheus/ginprometheus.go @@ -14,430 +14,431 @@ package ginprometheus -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var defaultMetricPath = "/metrics" - -// counter, counter_vec, gauge, gauge_vec, -// histogram, histogram_vec, summary, summary_vec. -var ( - reqCounter = &Metric{ - ID: "reqCnt", - Name: "requests_total", - Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", - Type: "counter_vec", - Args: []string{"code", "method", "handler", "host", "url"}} - - reqDuration = &Metric{ - ID: "reqDur", - Name: "request_duration_seconds", - Description: "The HTTP request latencies in seconds.", - Type: "histogram_vec", - Args: []string{"code", "method", "url"}, - } - - resSize = &Metric{ - ID: "resSz", - Name: "response_size_bytes", - Description: "The HTTP response sizes in bytes.", - Type: "summary"} - - reqSize = &Metric{ - ID: "reqSz", - Name: "request_size_bytes", - Description: "The HTTP request sizes in bytes.", - Type: "summary"} - - standardMetrics = []*Metric{ - reqCounter, - reqDuration, - resSize, - reqSize, - } -) - -/* -RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control -the cardinality of the request counter's "url" label, which might be required in some contexts. -For instance, if for a "/customer/:name" route you don't want to generate a time series for every -possible customer name, you could use this function: - - func(c *gin.Context) string { - url := c.Request.URL.Path - for _, p := range c.Params { - if p.Key == "name" { - url = strings.Replace(url, p.Value, ":name", 1) - break - } - } - return url - } - -which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". -*/ -type RequestCounterURLLabelMappingFn func(c *gin.Context) string - -// Metric is a definition for the name, description, type, ID, and -// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. -type Metric struct { - MetricCollector prometheus.Collector - ID string - Name string - Description string - Type string - Args []string -} - -// Prometheus contains the metrics gathered by the instance and its path. -type Prometheus struct { - reqCnt *prometheus.CounterVec - reqDur *prometheus.HistogramVec - reqSz, resSz prometheus.Summary - router *gin.Engine - listenAddress string - Ppg PrometheusPushGateway - - MetricsList []*Metric - MetricsPath string - - ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn - - // gin.Context string to use as a prometheus URL label - URLLabelFromContext string -} - -// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). -type PrometheusPushGateway struct { - - // Push interval in seconds - PushIntervalSeconds time.Duration - - // Push Gateway URL in format http://domain:port - // where JOBNAME can be any string of your choice - PushGatewayURL string - - // Local metrics URL where metrics are fetched from, this could be omitted in the future - // if implemented using prometheus common/expfmt instead - MetricsURL string - - // pushgateway job name, defaults to "gin" - Job string -} - -// NewPrometheus generates a new set of metrics with a certain subsystem name. -func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { - if subsystem == "" { - subsystem = "app" - } - - var metricsList []*Metric - - if len(customMetricsList) > 1 { - panic("Too many args. NewPrometheus( string, ).") - } else if len(customMetricsList) == 1 { - metricsList = customMetricsList[0] - } - metricsList = append(metricsList, standardMetrics...) - - p := &Prometheus{ - MetricsList: metricsList, - MetricsPath: defaultMetricPath, - ReqCntURLLabelMappingFn: func(c *gin.Context) string { - return c.FullPath() // e.g. /user/:id , /user/:id/info - }, - } - - p.registerMetrics(subsystem) - - return p -} - -// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL -// every pushIntervalSeconds. Metrics are fetched from metricsURL. -func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { - p.Ppg.PushGatewayURL = pushGatewayURL - p.Ppg.MetricsURL = metricsURL - p.Ppg.PushIntervalSeconds = pushIntervalSeconds - p.startPushTicker() -} - -// SetPushGatewayJob job name, defaults to "gin". -func (p *Prometheus) SetPushGatewayJob(j string) { - p.Ppg.Job = j -} - -// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the -// same address of the gin engine that is being used. -func (p *Prometheus) SetListenAddress(address string) { - p.listenAddress = address - if p.listenAddress != "" { - p.router = gin.Default() - } -} - -// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of -// your content's access log). -func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { - p.listenAddress = listenAddress - if len(p.listenAddress) > 0 { - p.router = r - } -} - -// SetMetricsPath set metrics paths. -func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, prometheusHandler()) - return nil - } -} - -// SetMetricsPathWithAuth set metrics paths with authentication. -func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return nil - } - -} - -func (p *Prometheus) runServer() error { - return p.router.Run(p.listenAddress) -} - -func (p *Prometheus) getMetrics() []byte { - response, err := http.Get(p.Ppg.MetricsURL) - if err != nil { - return nil - } - - defer response.Body.Close() - - body, _ := io.ReadAll(response.Body) - return body -} - -var hostname, _ = os.Hostname() - -func (p *Prometheus) getPushGatewayURL() string { - if p.Ppg.Job == "" { - p.Ppg.Job = "gin" - } - return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname -} - -func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { - req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) - if err != nil { - return - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - fmt.Println("Error sending to push gateway error:", err.Error()) - } - - resp.Body.Close() -} - -func (p *Prometheus) startPushTicker() { - ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) - go func() { - for range ticker.C { - p.sendMetricsToPushGateway(p.getMetrics()) - } - }() -} - -// NewMetric associates prometheus.Collector based on Metric.Type. -func NewMetric(m *Metric, subsystem string) prometheus.Collector { - var metric prometheus.Collector - switch m.Type { - case "counter_vec": - metric = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "counter": - metric = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "gauge_vec": - metric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "gauge": - metric = prometheus.NewGauge( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "histogram_vec": - metric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "histogram": - metric = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "summary_vec": - metric = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "summary": - metric = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - } - return metric -} - -func (p *Prometheus) registerMetrics(subsystem string) { - for _, metricDef := range p.MetricsList { - metric := NewMetric(metricDef, subsystem) - if err := prometheus.Register(metric); err != nil { - fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) - } - - switch metricDef { - case reqCounter: - p.reqCnt = metric.(*prometheus.CounterVec) - case reqDuration: - p.reqDur = metric.(*prometheus.HistogramVec) - case resSize: - p.resSz = metric.(prometheus.Summary) - case reqSize: - p.reqSz = metric.(prometheus.Summary) - } - metricDef.MetricCollector = metric - } -} - -// Use adds the middleware to a gin engine. -func (p *Prometheus) Use(e *gin.Engine) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPath(e) -} - -// UseWithAuth adds the middleware to a gin engine with BasicAuth. -func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPathWithAuth(e, accounts) -} - -// HandlerFunc defines handler function for middleware. -func (p *Prometheus) HandlerFunc() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.URL.Path == p.MetricsPath { - c.Next() - return - } - - start := time.Now() - reqSz := computeApproximateRequestSize(c.Request) - - c.Next() - - status := strconv.Itoa(c.Writer.Status()) - elapsed := float64(time.Since(start)) / float64(time.Second) - resSz := float64(c.Writer.Size()) - - url := p.ReqCntURLLabelMappingFn(c) - if len(p.URLLabelFromContext) > 0 { - u, found := c.Get(p.URLLabelFromContext) - if !found { - u = "unknown" - } - url = u.(string) - } - p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) - p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() - p.reqSz.Observe(float64(reqSz)) - p.resSz.Observe(resSz) - } -} - -func prometheusHandler() gin.HandlerFunc { - h := promhttp.Handler() - return func(c *gin.Context) { - h.ServeHTTP(c.Writer, c.Request) - } -} - -func computeApproximateRequestSize(r *http.Request) int { - var s int - if r.URL != nil { - s = len(r.URL.Path) - } - - s += len(r.Method) - s += len(r.Proto) - for name, values := range r.Header { - s += len(name) - for _, value := range values { - s += len(value) - } - } - s += len(r.Host) - - // r.FormData and r.MultipartForm are assumed to be included in r.URL. - - if r.ContentLength != -1 { - s += int(r.ContentLength) - } - return s -} +// +//import ( +// "bytes" +// "fmt" +// "io" +// "net/http" +// "os" +// "strconv" +// "time" +// +// "github.com/gin-gonic/gin" +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" +//) +// +//var defaultMetricPath = "/metrics" +// +//// counter, counter_vec, gauge, gauge_vec, +//// histogram, histogram_vec, summary, summary_vec. +//var ( +// reqCounter = &Metric{ +// ID: "reqCnt", +// Name: "requests_total", +// Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", +// Type: "counter_vec", +// Args: []string{"code", "method", "handler", "host", "url"}} +// +// reqDuration = &Metric{ +// ID: "reqDur", +// Name: "request_duration_seconds", +// Description: "The HTTP request latencies in seconds.", +// Type: "histogram_vec", +// Args: []string{"code", "method", "url"}, +// } +// +// resSize = &Metric{ +// ID: "resSz", +// Name: "response_size_bytes", +// Description: "The HTTP response sizes in bytes.", +// Type: "summary"} +// +// reqSize = &Metric{ +// ID: "reqSz", +// Name: "request_size_bytes", +// Description: "The HTTP request sizes in bytes.", +// Type: "summary"} +// +// standardMetrics = []*Metric{ +// reqCounter, +// reqDuration, +// resSize, +// reqSize, +// } +//) +// +///* +//RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control +//the cardinality of the request counter's "url" label, which might be required in some contexts. +//For instance, if for a "/customer/:name" route you don't want to generate a time series for every +//possible customer name, you could use this function: +// +// func(c *gin.Context) string { +// url := c.Request.URL.Path +// for _, p := range c.Params { +// if p.Key == "name" { +// url = strings.Replace(url, p.Value, ":name", 1) +// break +// } +// } +// return url +// } +// +//which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". +//*/ +//type RequestCounterURLLabelMappingFn func(c *gin.Context) string +// +//// Metric is a definition for the name, description, type, ID, and +//// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. +//type Metric struct { +// MetricCollector prometheus.Collector +// ID string +// Name string +// Description string +// Type string +// Args []string +//} +// +//// Prometheus contains the metrics gathered by the instance and its path. +//type Prometheus struct { +// reqCnt *prometheus.CounterVec +// reqDur *prometheus.HistogramVec +// reqSz, resSz prometheus.Summary +// router *gin.Engine +// listenAddress string +// Ppg PrometheusPushGateway +// +// MetricsList []*Metric +// MetricsPath string +// +// ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn +// +// // gin.Context string to use as a prometheus URL label +// URLLabelFromContext string +//} +// +//// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). +//type PrometheusPushGateway struct { +// +// // Push interval in seconds +// PushIntervalSeconds time.Duration +// +// // Push Gateway URL in format http://domain:port +// // where JOBNAME can be any string of your choice +// PushGatewayURL string +// +// // Local metrics URL where metrics are fetched from, this could be omitted in the future +// // if implemented using prometheus common/expfmt instead +// MetricsURL string +// +// // pushgateway job name, defaults to "gin" +// Job string +//} +// +//// NewPrometheus generates a new set of metrics with a certain subsystem name. +//func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { +// if subsystem == "" { +// subsystem = "app" +// } +// +// var metricsList []*Metric +// +// if len(customMetricsList) > 1 { +// panic("Too many args. NewPrometheus( string, ).") +// } else if len(customMetricsList) == 1 { +// metricsList = customMetricsList[0] +// } +// metricsList = append(metricsList, standardMetrics...) +// +// p := &Prometheus{ +// MetricsList: metricsList, +// MetricsPath: defaultMetricPath, +// ReqCntURLLabelMappingFn: func(c *gin.Context) string { +// return c.FullPath() // e.g. /user/:id , /user/:id/info +// }, +// } +// +// p.registerMetrics(subsystem) +// +// return p +//} +// +//// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL +//// every pushIntervalSeconds. Metrics are fetched from metricsURL. +//func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { +// p.Ppg.PushGatewayURL = pushGatewayURL +// p.Ppg.MetricsURL = metricsURL +// p.Ppg.PushIntervalSeconds = pushIntervalSeconds +// p.startPushTicker() +//} +// +//// SetPushGatewayJob job name, defaults to "gin". +//func (p *Prometheus) SetPushGatewayJob(j string) { +// p.Ppg.Job = j +//} +// +//// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the +//// same address of the gin engine that is being used. +//func (p *Prometheus) SetListenAddress(address string) { +// p.listenAddress = address +// if p.listenAddress != "" { +// p.router = gin.Default() +// } +//} +// +//// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of +//// your content's access log). +//func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { +// p.listenAddress = listenAddress +// if len(p.listenAddress) > 0 { +// p.router = r +// } +//} +// +//// SetMetricsPath set metrics paths. +//func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, prometheusHandler()) +// return nil +// } +//} +// +//// SetMetricsPathWithAuth set metrics paths with authentication. +//func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return nil +// } +// +//} +// +//func (p *Prometheus) runServer() error { +// return p.router.Run(p.listenAddress) +//} +// +//func (p *Prometheus) getMetrics() []byte { +// response, err := http.Get(p.Ppg.MetricsURL) +// if err != nil { +// return nil +// } +// +// defer response.Body.Close() +// +// body, _ := io.ReadAll(response.Body) +// return body +//} +// +//var hostname, _ = os.Hostname() +// +//func (p *Prometheus) getPushGatewayURL() string { +// if p.Ppg.Job == "" { +// p.Ppg.Job = "gin" +// } +// return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname +//} +// +//func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { +// req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) +// if err != nil { +// return +// } +// +// client := &http.Client{} +// resp, err := client.Do(req) +// if err != nil { +// fmt.Println("Error sending to push gateway error:", err.Error()) +// } +// +// resp.Body.Close() +//} +// +//func (p *Prometheus) startPushTicker() { +// ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) +// go func() { +// for range ticker.C { +// p.sendMetricsToPushGateway(p.getMetrics()) +// } +// }() +//} +// +//// NewMetric associates prometheus.Collector based on Metric.Type. +//func NewMetric(m *Metric, subsystem string) prometheus.Collector { +// var metric prometheus.Collector +// switch m.Type { +// case "counter_vec": +// metric = prometheus.NewCounterVec( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "counter": +// metric = prometheus.NewCounter( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "gauge_vec": +// metric = prometheus.NewGaugeVec( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "gauge": +// metric = prometheus.NewGauge( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "histogram_vec": +// metric = prometheus.NewHistogramVec( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "histogram": +// metric = prometheus.NewHistogram( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "summary_vec": +// metric = prometheus.NewSummaryVec( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "summary": +// metric = prometheus.NewSummary( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// } +// return metric +//} +// +//func (p *Prometheus) registerMetrics(subsystem string) { +// for _, metricDef := range p.MetricsList { +// metric := NewMetric(metricDef, subsystem) +// if err := prometheus.Register(metric); err != nil { +// fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) +// } +// +// switch metricDef { +// case reqCounter: +// p.reqCnt = metric.(*prometheus.CounterVec) +// case reqDuration: +// p.reqDur = metric.(*prometheus.HistogramVec) +// case resSize: +// p.resSz = metric.(prometheus.Summary) +// case reqSize: +// p.reqSz = metric.(prometheus.Summary) +// } +// metricDef.MetricCollector = metric +// } +//} +// +//// Use adds the middleware to a gin engine. +//func (p *Prometheus) Use(e *gin.Engine) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPath(e) +//} +// +//// UseWithAuth adds the middleware to a gin engine with BasicAuth. +//func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPathWithAuth(e, accounts) +//} +// +//// HandlerFunc defines handler function for middleware. +//func (p *Prometheus) HandlerFunc() gin.HandlerFunc { +// return func(c *gin.Context) { +// if c.Request.URL.Path == p.MetricsPath { +// c.Next() +// return +// } +// +// start := time.Now() +// reqSz := computeApproximateRequestSize(c.Request) +// +// c.Next() +// +// status := strconv.Itoa(c.Writer.Status()) +// elapsed := float64(time.Since(start)) / float64(time.Second) +// resSz := float64(c.Writer.Size()) +// +// url := p.ReqCntURLLabelMappingFn(c) +// if len(p.URLLabelFromContext) > 0 { +// u, found := c.Get(p.URLLabelFromContext) +// if !found { +// u = "unknown" +// } +// url = u.(string) +// } +// p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) +// p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() +// p.reqSz.Observe(float64(reqSz)) +// p.resSz.Observe(resSz) +// } +//} +// +//func prometheusHandler() gin.HandlerFunc { +// h := promhttp.Handler() +// return func(c *gin.Context) { +// h.ServeHTTP(c.Writer, c.Request) +// } +//} +// +//func computeApproximateRequestSize(r *http.Request) int { +// var s int +// if r.URL != nil { +// s = len(r.URL.Path) +// } +// +// s += len(r.Method) +// s += len(r.Proto) +// for name, values := range r.Header { +// s += len(name) +// for _, value := range values { +// s += len(value) +// } +// } +// s += len(r.Host) +// +// // r.FormData and r.MultipartForm are assumed to be included in r.URL. +// +// if r.ContentLength != -1 { +// s += int(r.ContentLength) +// } +// return s +//} diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go new file mode 100644 index 0000000000..ce45290add --- /dev/null +++ b/pkg/common/prommetrics/api.go @@ -0,0 +1,106 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "strconv" + "time" +) + +const ApiPath = "/metrics" + +var ( + apiRegistry = prometheus.NewRegistry() + apiCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_count", + Help: "Total number of API calls", + }, + []string{"path", "code", "type"}, + ) + httpCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_count", + Help: "Total number of HTTP calls", + }, + []string{"path", "method", "status", "duration"}, + ) +) + +func init() { + apiRegistry.MustRegister(apiCounter, httpCounter) +} + +func APICall(path string, apiCode int, apiType string) { + apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() +} + +func HttpCall(path string, method string, status int, duration time.Duration) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +} + +var ( + durations = [...]time.Duration{ + time.Millisecond * 1, + time.Millisecond * 2, + time.Millisecond * 3, + time.Millisecond * 4, + time.Millisecond * 5, + time.Millisecond * 6, + time.Millisecond * 7, + time.Millisecond * 8, + time.Millisecond * 9, + time.Millisecond * 10, + time.Millisecond * 20, + time.Millisecond * 30, + time.Millisecond * 40, + time.Millisecond * 50, + time.Millisecond * 60, + time.Millisecond * 70, + time.Millisecond * 80, + time.Millisecond * 90, + time.Millisecond * 100, + time.Millisecond * 200, + time.Millisecond * 300, + time.Millisecond * 400, + time.Millisecond * 500, + time.Millisecond * 600, + time.Millisecond * 700, + time.Millisecond * 800, + time.Millisecond * 900, + time.Second * 1, + time.Second * 2, + time.Second * 3, + time.Second * 4, + time.Second * 5, + time.Second * 6, + time.Second * 7, + time.Second * 8, + time.Second * 9, + time.Second * 10, + time.Second * 20, + time.Second * 30, + time.Second * 40, + time.Second * 50, + time.Second * 60, + time.Second * 70, + time.Second * 80, + time.Second * 90, + time.Second * 100, + } + maxDuration = durations[len(durations)-1] +) + +func durationRange(duration time.Duration) string { + for _, d := range durations { + if duration <= d { + return d.String() + } + } + return ">" + maxDuration.String() +} + +func ApiHandler() http.Handler { + return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) +} diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go new file mode 100644 index 0000000000..3a691c7949 --- /dev/null +++ b/pkg/common/prommetrics/api_test.go @@ -0,0 +1,56 @@ +package prommetrics + +import ( + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "testing" + "time" +) + +var ( + apiCallCounter1 = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_calls_total", + Help: "Total number of API calls", + }, + []string{"endpoint", "status", "code", "error"}, + ) + registerer *prometheus.Registry +) + +func init() { + registerer = prometheus.NewRegistry() + registerer.MustRegister(apiCallCounter1) +} + +func recordAPICall(endpoint string, status string) { + apiCallCounter1.With(prometheus.Labels{"endpoint": endpoint, "status": status, "code": "200", "error": "ArgsError"}).Inc() +} + +func TestName(t *testing.T) { + go func() { + for i := 0; ; i++ { + recordAPICall("/api/test", "success") + time.Sleep(time.Second) + } + }() + + go func() { + for i := 0; ; i++ { + recordAPICall("/api/test", "failed") + time.Sleep(time.Second * 3) + } + }() + http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) + if err := http.ListenAndServe(":2112", nil); err != nil { + panic(err) + } +} + +func TestName2(t *testing.T) { + var d time.Duration + d = time.Second / 900 + fmt.Println(durationRange(d)) +} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go index 9f2e4c99d7..47b043f64b 100644 --- a/pkg/common/prommetrics/gin_api.go +++ b/pkg/common/prommetrics/gin_api.go @@ -14,17 +14,17 @@ package prommetrics -import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" +//import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" /* labels := prometheus.Labels{"label_one": "any", "label_two": "value"} ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). */ -var ( - ApiCustomCnt = &ginprom.Metric{ - Name: "custom_total", - Description: "Custom counter events.", - Type: "counter_vec", - Args: []string{"label_one", "label_two"}, - } -) +//var ( +// ApiCustomCnt = &ginprom.Metric{ +// Name: "custom_total", +// Description: "Custom counter events.", +// Type: "counter_vec", +// Args: []string{"label_one", "label_two"}, +// } +//) diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 47e5d02b88..3955d8ea5f 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -17,7 +17,6 @@ package prommetrics import ( gp "github.com/grpc-ecosystem/go-grpc-prometheus" config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" ) @@ -48,11 +47,11 @@ func GetGrpcCusMetrics(registerName string, share *config2.Share) []prometheus.C } } -func GetGinCusMetrics(name string) []*ginprometheus.Metric { - switch name { - case "Api": - return []*ginprometheus.Metric{ApiCustomCnt} - default: - return []*ginprometheus.Metric{ApiCustomCnt} - } -} +//func GetGinCusMetrics(name string) []*ginprometheus.Metric { +// switch name { +// case "Api": +// return []*ginprometheus.Metric{ApiCustomCnt} +// default: +// return []*ginprometheus.Metric{ApiCustomCnt} +// } +//} From 671398688714dc754839c723066f0c6c2aad7a9d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:15:27 +0800 Subject: [PATCH 097/147] api prommetrics --- internal/api/router.go | 13 ++++++------- pkg/common/prommetrics/api.go | 11 ++++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index b519c30ff2..936d6ab0b2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,14 +2,10 @@ package api import ( "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "net/http" - "strings" - "time" - "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -19,14 +15,17 @@ import ( "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "net/http" + "strings" ) func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { - start := time.Now() + //start := time.Now() c.Next() path := c.FullPath() - prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) if c.Request.Method == http.MethodPost { if resp := apiresp.GetGinApiResponse(c); resp == nil { prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index ce45290add..aa5e0a60e4 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -24,7 +24,8 @@ var ( Name: "http_count", Help: "Total number of HTTP calls", }, - []string{"path", "method", "status", "duration"}, + //[]string{"path", "method", "status", "duration"}, + []string{"path", "method", "status"}, ) ) @@ -36,8 +37,12 @@ func APICall(path string, apiCode int, apiType string) { apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() } -func HttpCall(path string, method string, status int, duration time.Duration) { - httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +//func HttpCall(path string, method string, status int, duration time.Duration) { +// httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +//} + +func HttpCall(path string, method string, status int) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } var ( From 9151b56fe9b1eb2ea52b92801ac183f2f5c30cd6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:21:24 +0800 Subject: [PATCH 098/147] api prommetrics --- internal/api/router.go | 4 ++-- pkg/common/prommetrics/api.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 936d6ab0b2..1c70c6bc58 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -28,9 +28,9 @@ func prommetricsGin() gin.HandlerFunc { //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) if c.Request.Method == http.MethodPost { if resp := apiresp.GetGinApiResponse(c); resp == nil { - prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") + prommetrics.APICall(path, -1) } else { - prommetrics.APICall(path, resp.ErrCode, resp.ErrMsg) + prommetrics.APICall(path, resp.ErrCode) } } } diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index aa5e0a60e4..15f8a3e319 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -17,7 +17,7 @@ var ( Name: "api_count", Help: "Total number of API calls", }, - []string{"path", "code", "type"}, + []string{"path", "code"}, ) httpCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -33,8 +33,8 @@ func init() { apiRegistry.MustRegister(apiCounter, httpCounter) } -func APICall(path string, apiCode int, apiType string) { - apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() +func APICall(path string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode)}).Inc() } //func HttpCall(path string, method string, status int, duration time.Duration) { From c1d66faa109199cb7695ddc0cb37f2d8668e213a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:36:20 +0800 Subject: [PATCH 099/147] api prommetrics --- internal/api/router.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 1c70c6bc58..0a42d82bd0 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -21,17 +21,15 @@ import ( func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { - //start := time.Now() c.Next() path := c.FullPath() - prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) - //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) - if c.Request.Method == http.MethodPost { - if resp := apiresp.GetGinApiResponse(c); resp == nil { - prommetrics.APICall(path, -1) - } else { - prommetrics.APICall(path, resp.ErrCode) - } + if c.Writer.Status() == http.StatusNotFound { + prommetrics.HttpCall("<404>", c.Request.Method, c.Writer.Status()) + } else { + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + } + if resp := apiresp.GetGinApiResponse(c); resp != nil { + prommetrics.APICall(path, c.Request.Method, resp.ErrCode) } } } From e3ee24df3129313ec6d9420c975bb46b786b6cc9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:40:09 +0800 Subject: [PATCH 100/147] api prommetrics --- pkg/common/prommetrics/api.go | 77 +++--------------------------- pkg/common/prommetrics/api_test.go | 1 + 2 files changed, 7 insertions(+), 71 deletions(-) diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 15f8a3e319..b80b8df854 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -5,7 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" "strconv" - "time" ) const ApiPath = "/metrics" @@ -17,14 +16,13 @@ var ( Name: "api_count", Help: "Total number of API calls", }, - []string{"path", "code"}, + []string{"path", "method", "code"}, ) httpCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "http_count", Help: "Total number of HTTP calls", }, - //[]string{"path", "method", "status", "duration"}, []string{"path", "method", "status"}, ) ) @@ -33,79 +31,16 @@ func init() { apiRegistry.MustRegister(apiCounter, httpCounter) } -func APICall(path string, apiCode int) { - apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode)}).Inc() +func APICall(path string, method string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "method": method, "code": strconv.Itoa(apiCode)}).Inc() } -//func HttpCall(path string, method string, status int, duration time.Duration) { -// httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() -//} - func HttpCall(path string, method string, status int) { httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } -var ( - durations = [...]time.Duration{ - time.Millisecond * 1, - time.Millisecond * 2, - time.Millisecond * 3, - time.Millisecond * 4, - time.Millisecond * 5, - time.Millisecond * 6, - time.Millisecond * 7, - time.Millisecond * 8, - time.Millisecond * 9, - time.Millisecond * 10, - time.Millisecond * 20, - time.Millisecond * 30, - time.Millisecond * 40, - time.Millisecond * 50, - time.Millisecond * 60, - time.Millisecond * 70, - time.Millisecond * 80, - time.Millisecond * 90, - time.Millisecond * 100, - time.Millisecond * 200, - time.Millisecond * 300, - time.Millisecond * 400, - time.Millisecond * 500, - time.Millisecond * 600, - time.Millisecond * 700, - time.Millisecond * 800, - time.Millisecond * 900, - time.Second * 1, - time.Second * 2, - time.Second * 3, - time.Second * 4, - time.Second * 5, - time.Second * 6, - time.Second * 7, - time.Second * 8, - time.Second * 9, - time.Second * 10, - time.Second * 20, - time.Second * 30, - time.Second * 40, - time.Second * 50, - time.Second * 60, - time.Second * 70, - time.Second * 80, - time.Second * 90, - time.Second * 100, - } - maxDuration = durations[len(durations)-1] -) - -func durationRange(duration time.Duration) string { - for _, d := range durations { - if duration <= d { - return d.String() - } - } - return ">" + maxDuration.String() -} - func ApiHandler() http.Handler { - return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) + return promhttp.InstrumentMetricHandler( + apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), + ) } diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go index 3a691c7949..b0e0f7b20f 100644 --- a/pkg/common/prommetrics/api_test.go +++ b/pkg/common/prommetrics/api_test.go @@ -43,6 +43,7 @@ func TestName(t *testing.T) { time.Sleep(time.Second * 3) } }() + promhttp.Handler() http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) if err := http.ListenAndServe(":2112", nil); err != nil { panic(err) From 0a9b53bbe313140699808275895eead9bbae96dd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 14:19:34 +0800 Subject: [PATCH 101/147] rpc prommetrics --- internal/api/init.go | 2 +- pkg/common/prommetrics/api.go | 10 ++++-- pkg/common/prommetrics/api_test.go | 57 ------------------------------ pkg/common/prommetrics/gin_api.go | 30 ---------------- pkg/common/prommetrics/rpc.go | 33 +++++++++++++++++ pkg/common/startrpc/start.go | 51 ++++++++++++++++++-------- 6 files changed, 78 insertions(+), 105 deletions(-) delete mode 100644 pkg/common/prommetrics/api_test.go delete mode 100644 pkg/common/prommetrics/gin_api.go create mode 100644 pkg/common/prommetrics/rpc.go diff --git a/internal/api/init.go b/internal/api/init.go index e6fe46d22d..679d901f4b 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -74,7 +74,7 @@ func Start(ctx context.Context, index int, config *Config) error { srv := http.NewServeMux() srv.Handle(prommetrics.ApiPath, prommetrics.ApiHandler()) if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) + netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } }() diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index b80b8df854..8061989770 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -39,8 +39,12 @@ func HttpCall(path string, method string, status int) { httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } +//func ApiHandler() http.Handler { +// return promhttp.InstrumentMetricHandler( +// apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), +// ) +//} + func ApiHandler() http.Handler { - return promhttp.InstrumentMetricHandler( - apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), - ) + return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) } diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go deleted file mode 100644 index b0e0f7b20f..0000000000 --- a/pkg/common/prommetrics/api_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package prommetrics - -import ( - "fmt" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" - "testing" - "time" -) - -var ( - apiCallCounter1 = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "api_calls_total", - Help: "Total number of API calls", - }, - []string{"endpoint", "status", "code", "error"}, - ) - registerer *prometheus.Registry -) - -func init() { - registerer = prometheus.NewRegistry() - registerer.MustRegister(apiCallCounter1) -} - -func recordAPICall(endpoint string, status string) { - apiCallCounter1.With(prometheus.Labels{"endpoint": endpoint, "status": status, "code": "200", "error": "ArgsError"}).Inc() -} - -func TestName(t *testing.T) { - go func() { - for i := 0; ; i++ { - recordAPICall("/api/test", "success") - time.Sleep(time.Second) - } - }() - - go func() { - for i := 0; ; i++ { - recordAPICall("/api/test", "failed") - time.Sleep(time.Second * 3) - } - }() - promhttp.Handler() - http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) - if err := http.ListenAndServe(":2112", nil); err != nil { - panic(err) - } -} - -func TestName2(t *testing.T) { - var d time.Duration - d = time.Second / 900 - fmt.Println(durationRange(d)) -} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go deleted file mode 100644 index 47b043f64b..0000000000 --- a/pkg/common/prommetrics/gin_api.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prommetrics - -//import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" - -/* -labels := prometheus.Labels{"label_one": "any", "label_two": "value"} -ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). -*/ -//var ( -// ApiCustomCnt = &ginprom.Metric{ -// Name: "custom_total", -// Description: "Custom counter events.", -// Type: "counter_vec", -// Args: []string{"label_one", "label_two"}, -// } -//) diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go new file mode 100644 index 0000000000..834f76a9fb --- /dev/null +++ b/pkg/common/prommetrics/rpc.go @@ -0,0 +1,33 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "strconv" +) + +const RpcPath = "/metrics" + +var ( + rpcRegistry = prometheus.NewRegistry() + rpcCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_count", + Help: "Total number of RPC calls", + }, + []string{"name", "path", "code"}, + ) +) + +func init() { + rpcRegistry.MustRegister(rpcCounter) +} + +func RPCCall(name string, path string, code int) { + rpcCounter.With(prometheus.Labels{"name": name, "path": path, "code": strconv.Itoa(code)}).Inc() +} + +func RPCHandler() http.Handler { + return promhttp.HandlerFor(rpcRegistry, promhttp.HandlerOpts{}) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index b531daa47d..a88797793c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -19,7 +19,7 @@ import ( "fmt" config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" - "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" "net" "net/http" "os" @@ -29,7 +29,6 @@ import ( "syscall" "time" - grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -38,7 +37,6 @@ import ( "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/network" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -77,13 +75,14 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC return err } - var reg *prometheus.Registry - var metric *grpcprometheus.ServerMetrics + //var reg *prometheus.Registry + //var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append(options, mw.GrpcServer(), prommetricsInterceptor(rpcRegisterName)) } else { options = append(options, mw.GrpcServer()) } @@ -122,13 +121,19 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC netDone <- struct{}{} return } - metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + srv := http.NewServeMux() + srv.Handle(prommetrics.RpcPath, prommetrics.RPCHandler()) + if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } + //metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + //} }() } @@ -175,3 +180,21 @@ func gracefulStopWithCtx(ctx context.Context, f func()) error { return nil } } + +func prommetricsInterceptor(rpcRegisterName string) grpc.ServerOption { + getCode := func(err error) int { + if err == nil { + return 0 + } + rpcErr, ok := err.(interface{ GRPCStatus() *status.Status }) + if !ok { + return -1 + } + return int(rpcErr.GRPCStatus().Code()) + } + return grpc.ChainUnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + resp, err := handler(ctx, req) + prommetrics.RPCCall(rpcRegisterName, info.FullMethod, getCode(err)) + return resp, err + }) +} From 2052e1c7d569be86c3af6b67c4030f35e4d532de Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 14:59:41 +0800 Subject: [PATCH 102/147] rpc prommetrics --- go.sum | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/go.sum b/go.sum index a5a620a144..577154a1b6 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -222,12 +220,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= -github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= @@ -270,10 +262,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= -github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= +github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -527,6 +519,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From c1967a63caf1af009c3312c407f667c87e093c5a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 18:47:48 +0800 Subject: [PATCH 103/147] online status --- internal/msggateway/init.go | 8 ++++++++ internal/msggateway/subscription.go | 25 ++++++++++++++++++++++++ internal/msggateway/user_map.go | 30 +++++++++++++++++++++++++++++ pkg/common/cmd/msg_gateway.go | 1 + pkg/rpccache/online.go | 24 ++--------------------- pkg/util/useronline/split.go | 27 ++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 22 deletions(-) create mode 100644 pkg/util/useronline/split.go diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 815ec8ca60..739a71ecba 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,7 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -26,6 +27,7 @@ import ( type Config struct { MsgGateway config.MsgGateway Share config.Share + RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery } @@ -42,6 +44,10 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) + if err != nil { + return err + } longServer := NewWsServer( conf, WithPort(wsPort), @@ -52,6 +58,8 @@ func Start(ctx context.Context, index int, conf *Config) error { go longServer.ChangeOnlineStatus(4) + go longServer.SubscriberUserOnlineStatusChanges(rdb) + hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) go func() { diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index d7e6060370..98cf41366a 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -1,5 +1,30 @@ package msggateway +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" + "math/rand" + "strconv" +) + +func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) { + ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) + for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + continue + } + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "receive subscription message and go back online", "userID", userID) + } + } +} + //import ( // "context" // "encoding/json" diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 57fa78087a..bd1f197281 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -13,6 +13,7 @@ type UserMap interface { DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState + RecvSubChange(userID string, platformIDs []int32) bool } type UserState struct { @@ -37,6 +38,17 @@ func (u *UserPlatform) PlatformIDs() []int32 { return platformIDs } +func (u *UserPlatform) PlatformIDSet() map[int32]struct{} { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make(map[int32]struct{}) + for _, client := range u.Clients { + platformIDs[int32(client.PlatformID)] = struct{}{} + } + return platformIDs +} + func newUserMap() UserMap { return &userMap{ data: make(map[string]*UserPlatform), @@ -50,6 +62,24 @@ type userMap struct { ch chan UserState } +func (u *userMap) RecvSubChange(userID string, platformIDs []int32) bool { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return false + } + localPlatformIDs := result.PlatformIDSet() + for _, platformID := range platformIDs { + delete(localPlatformIDs, platformID) + } + if len(localPlatformIDs) == 0 { + return false + } + u.push(userID, result, nil) + return true +} + func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 78004094c7..29d3fba330 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -37,6 +37,7 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { ret.configMap = map[string]any{ OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, ShareFileName: &msgGatewayConfig.Share, + RedisConfigFileName: &msgGatewayConfig.RedisConfig, WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index eac2f43af8..d986651c7d 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -2,17 +2,16 @@ package rpccache import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/redis/go-redis/v9" "math/rand" "strconv" - "strings" "time" ) @@ -25,28 +24,9 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re }), } go func() { - parseUserOnlineStatus := func(payload string) (string, []int32, error) { - arr := strings.Split(payload, ":") - if len(arr) == 0 { - return "", nil, errors.New("invalid data") - } - userID := arr[len(arr)-1] - if userID == "" { - return "", nil, errors.New("userID is empty") - } - platformIDs := make([]int32, len(arr)-1) - for i := range platformIDs { - platformID, err := strconv.Atoi(arr[i]) - if err != nil { - return "", nil, err - } - platformIDs[i] = int32(platformID) - } - return userID, platformIDs, nil - } ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := parseUserOnlineStatus(message.Payload) + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) if err != nil { log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue diff --git a/pkg/util/useronline/split.go b/pkg/util/useronline/split.go new file mode 100644 index 0000000000..c39d31d15b --- /dev/null +++ b/pkg/util/useronline/split.go @@ -0,0 +1,27 @@ +package useronline + +import ( + "errors" + "strconv" + "strings" +) + +func ParseUserOnlineStatus(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil +} From b53a26d2cadf6eab3b2ef7e834c145c0dcd63489 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 10:04:31 +0800 Subject: [PATCH 104/147] online status --- internal/msggateway/subscription.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 98cf41366a..8df548f543 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -20,7 +20,9 @@ func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) continue } if ws.clients.RecvSubChange(userID, platformIDs) { - log.ZDebug(ctx, "receive subscription message and go back online", "userID", userID) + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) } } } From fd2b0b32843b6e1316cf6507e90fd19293be0a8e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 10:11:40 +0800 Subject: [PATCH 105/147] online status --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 5bc9f6d984..577154a1b6 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 67d17c82d519b9372136ed46e5ea0a4cedc9b301 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 11:09:03 +0800 Subject: [PATCH 106/147] online status --- pkg/common/storage/cache/redis/user.go | 100 ------------------------- 1 file changed, 100 deletions(-) diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index b846bf3981..c3accd2c32 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -128,106 +128,6 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } -/* - - */ - -type RedisUserOnline struct { - // 平台id, 平台更新时间 - PlatformIDs map[int32]int64 -} - -// GetUserStatus get user status. -//func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { -// userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) -// for _, userID := range userIDs { -// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) -// modKey := strconv.Itoa(int(UserIDNum % statusMod)) -// var onlineStatus user.OnlineStatus -// key := u.getOnlineStatusKey(modKey) -// result, err := u.rdb.HGet(ctx, key, userID).Result() -// if err != nil { -// if errors.Is(err, redis.Nil) { -// // key or field does not exist -// userStatus = append(userStatus, &user.OnlineStatus{ -// UserID: userID, -// Status: constant.Offline, -// PlatformIDs: nil, -// }) -// -// continue -// } else { -// return nil, errs.Wrap(err) -// } -// } -// err = json.Unmarshal([]byte(result), &onlineStatus) -// if err != nil { -// return nil, errs.Wrap(err) -// } -// onlineStatus.UserID = userID -// onlineStatus.Status = constant.Online -// userStatus = append(userStatus, &onlineStatus) -// } -// -// return userStatus, nil -//} - -// SetUserStatus Set the user status and save it in redis. -//func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { -// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) -// modKey := strconv.Itoa(int(UserIDNum % statusMod)) -// key := u.getOnlineStatusKey(modKey) -// log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) -// isNewKey, err := u.rdb.Exists(ctx, key).Result() -// if err != nil { -// return errs.Wrap(err) -// } -// if isNewKey == 0 { -// if status == constant.Online { -// onlineStatus := user.OnlineStatus{ -// UserID: userID, -// Status: constant.Online, -// PlatformIDs: []int32{platformID}, -// } -// jsonData, err := json.Marshal(&onlineStatus) -// if err != nil { -// return errs.Wrap(err) -// } -// _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() -// if err != nil { -// return errs.Wrap(err) -// } -// u.rdb.Expire(ctx, key, userOlineStatusExpireTime) -// -// return nil -// } -// } -// -// isNil := false -// result, err := u.rdb.HGet(ctx, key, userID).Result() -// if err != nil { -// if errors.Is(err, redis.Nil) { -// isNil = true -// } else { -// return errs.Wrap(err) -// } -// } -// -// if status == constant.Offline { -// err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) -// if err != nil { -// return err -// } -// } else { -// err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) -// if err != nil { -// return errs.Wrap(err) -// } -// } -// -// return nil -//} - func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { if isNil { log.ZWarn(ctx, "this user not online,maybe trigger order not right", From 9a4f5f78cb03fa16261059a44d3d95d348121d41 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 9 Jul 2024 14:26:55 +0800 Subject: [PATCH 107/147] sub --- go.mod | 2 + go.sum | 2 - internal/msggateway/client.go | 6 +- internal/msggateway/compressor_test.go | 9 +- internal/msggateway/constant.go | 1 + internal/msggateway/hub_server.go | 10 +- internal/msggateway/init.go | 9 +- internal/msggateway/n_ws_server.go | 32 ++- internal/msggateway/subscription.go | 330 +++++++++++++------------ internal/push/push_handler.go | 2 +- pkg/rpccache/online.go | 5 +- 11 files changed, 222 insertions(+), 186 deletions(-) diff --git a/go.mod b/go.mod index 6450f36158..41ab0a322d 100644 --- a/go.mod +++ b/go.mod @@ -175,3 +175,5 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 577154a1b6..92350d7457 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,6 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= -github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 62c71d5be7..bed454e49d 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -72,8 +72,8 @@ type Client struct { closed atomic.Bool closedErr error token string - //subLock sync.Mutex - //subUserIDs map[string]struct{} + subLock sync.Mutex + subUserIDs map[string]struct{} } // ResetClient updates the client's state with new connection and context information. @@ -204,6 +204,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) + case WsSubUserOnlineStatus: + resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/compressor_test.go b/internal/msggateway/compressor_test.go index 173c9bb204..952bd4d95b 100644 --- a/internal/msggateway/compressor_test.go +++ b/internal/msggateway/compressor_test.go @@ -16,10 +16,10 @@ package msggateway import ( "crypto/rand" + "github.com/stretchr/testify/assert" "sync" "testing" - - "github.com/stretchr/testify/assert" + "unsafe" ) func mockRandom() []byte { @@ -132,3 +132,8 @@ func BenchmarkDecompressWithSyncPool(b *testing.B) { assert.Equal(b, nil, err) } } + +func TestName(t *testing.T) { + t.Log(unsafe.Sizeof(Client{})) + +} diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 64664ac0ab..bc74ed5833 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -43,6 +43,7 @@ const ( WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 + WsSubUserOnlineStatus = 2005 WSDataError = 3001 ) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8ff6d10018..3891aa532c 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/tools/discovery" @@ -31,6 +32,10 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + if s.ready != nil { + return s.ready(s) + } return nil } @@ -50,18 +55,21 @@ type Server struct { LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} + ready func(srv *Server) error + userRcp rpcclient.UserRpcClient } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config) *Server { +func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, + ready: ready, } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 739a71ecba..44e79e4122 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,7 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -56,11 +57,13 @@ func Start(ctx context.Context, index int, conf *Config) error { WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - go longServer.ChangeOnlineStatus(4) + hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + longServer.online = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, longServer.subscriberUserOnlineStatusChanges) + return nil + }) - go longServer.SubscriberUserOnlineStatusChanges(rdb) + go longServer.ChangeOnlineStatus(4) - hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) go func() { err = hubServer.Start(ctx, index, conf) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 8a79151fe7..e903084a9d 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/mcontext" "net/http" @@ -48,20 +49,22 @@ type LongConnServer interface { KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) + SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor Encoder MessageHandler } type WsServer struct { - msgGatewayConfig *Config - port int - wsMaxConnNum int64 - registerChan chan *Client - unregisterChan chan *Client - kickHandlerChan chan *kickHandler - clients UserMap - //subscription *Subscription + msgGatewayConfig *Config + port int + wsMaxConnNum int64 + registerChan chan *Client + unregisterChan chan *Client + kickHandlerChan chan *kickHandler + clients UserMap + online *rpccache.OnlineCache + subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -125,6 +128,8 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { for _, o := range opts { o(&config) } + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + v := validator.New() return &WsServer{ msgGatewayConfig: msgGatewayConfig, @@ -142,10 +147,10 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), - //subscription: newSubscription(), - Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), - webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), + subscription: newSubscription(), + Compressor: NewGzipCompressor(), + Encoder: NewGobEncoder(), + webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } @@ -353,6 +358,9 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) + client.subLock.Lock() + clear(client.subUserIDs) + client.subLock.Unlock() //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 8df548f543..9460f5dbfb 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -2,174 +2,180 @@ package msggateway import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "encoding/json" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/redis/go-redis/v9" - "math/rand" - "strconv" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/idutil" + "google.golang.org/protobuf/proto" + "sync" + "time" ) -func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) { - ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) - for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) - if err != nil { - log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) +func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + } + ws.pushUserIDOnlineStatus(ctx, userID, platformIDs) +} + +func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) { + var sub sdkws.SubUserOnlineStatus + if err := proto.Unmarshal(data.Data, &sub); err != nil { + return nil, err + } + ws.subscription.Sub(client, sub.SubscribeUserID, sub.UnsubscribeUserID) + var resp sdkws.SubUserOnlineStatusTips + if len(sub.SubscribeUserID) > 0 { + resp.Subscribers = make([]*sdkws.SubUserOnlineStatusElem, 0, len(sub.SubscribeUserID)) + for _, userID := range sub.SubscribeUserID { + platformIDs, err := ws.online.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + resp.Subscribers = append(resp.Subscribers, &sdkws.SubUserOnlineStatusElem{ + UserID: userID, + OnlinePlatformIDs: platformIDs, + }) + } + } + return proto.Marshal(&resp) +} + +type subClient struct { + clients map[string]*Client +} + +func newSubscription() *Subscription { + return &Subscription{ + userIDs: make(map[string]*subClient), + } +} + +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient +} + +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + +func (s *Subscription) DelClient(client *Client) { + client.subLock.Lock() + userIDs := datautil.Keys(client.subUserIDs) + for _, userID := range userIDs { + delete(client.subUserIDs, userID) + } + client.subLock.Unlock() + if len(userIDs) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for _, userID := range userIDs { + sub, ok := s.userIDs[userID] + if !ok { continue } - if ws.clients.RecvSubChange(userID, platformIDs) { - log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) - } else { - log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) } } } -//import ( -// "context" -// "encoding/json" -// "github.com/openimsdk/protocol/constant" -// "github.com/openimsdk/protocol/sdkws" -// "github.com/openimsdk/tools/log" -// "github.com/openimsdk/tools/utils/datautil" -// "github.com/openimsdk/tools/utils/idutil" -// "sync" -// "time" -//) -// -//type subClient struct { -// clients map[string]*Client -//} -// -//func newSubscription() *Subscription { -// return &Subscription{ -// userIDs: make(map[string]*subClient), -// } -//} -// -//type Subscription struct { -// lock sync.RWMutex -// userIDs map[string]*subClient -//} -// -//func (s *Subscription) GetClient(userID string) []*Client { -// s.lock.RLock() -// defer s.lock.RUnlock() -// cs, ok := s.userIDs[userID] -// if !ok { -// return nil -// } -// clients := make([]*Client, 0, len(cs.clients)) -// for _, client := range cs.clients { -// clients = append(clients, client) -// } -// return clients -//} -// -//func (s *Subscription) DelClient(client *Client) { -// client.subLock.Lock() -// userIDs := datautil.Keys(client.subUserIDs) -// for _, userID := range userIDs { -// delete(client.subUserIDs, userID) -// } -// client.subLock.Unlock() -// if len(userIDs) == 0 { -// return -// } -// addr := client.ctx.GetRemoteAddr() -// s.lock.Lock() -// defer s.lock.Unlock() -// for _, userID := range userIDs { -// sub, ok := s.userIDs[userID] -// if !ok { -// continue -// } -// delete(sub.clients, addr) -// if len(sub.clients) == 0 { -// delete(s.userIDs, userID) -// } -// } -//} -// -//func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { -// if len(addUserIDs)+len(delUserIDs) == 0 { -// return -// } -// var ( -// del = make(map[string]struct{}) -// add = make(map[string]struct{}) -// ) -// client.subLock.Lock() -// for _, userID := range delUserIDs { -// if _, ok := client.subUserIDs[userID]; !ok { -// continue -// } -// del[userID] = struct{}{} -// delete(client.subUserIDs, userID) -// } -// for _, userID := range addUserIDs { -// delete(del, userID) -// if _, ok := client.subUserIDs[userID]; ok { -// continue -// } -// client.subUserIDs[userID] = struct{}{} -// } -// client.subLock.Unlock() -// if len(del)+len(add) == 0 { -// return -// } -// addr := client.ctx.GetRemoteAddr() -// s.lock.Lock() -// defer s.lock.Unlock() -// for userID := range del { -// sub, ok := s.userIDs[userID] -// if !ok { -// continue -// } -// delete(sub.clients, addr) -// if len(sub.clients) == 0 { -// delete(s.userIDs, userID) -// } -// } -// for userID := range add { -// sub, ok := s.userIDs[userID] -// if !ok { -// sub = &subClient{clients: make(map[string]*Client)} -// s.userIDs[userID] = sub -// } -// sub.clients[addr] = client -// } -//} -// -//func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { -// clients := ws.subscription.GetClient(userID) -// if len(clients) == 0 { -// return -// } -// msgContent, err := json.Marshal(platformIDs) -// if err != nil { -// log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) -// return -// } -// now := time.Now().UnixMilli() -// msgID := idutil.GetMsgIDByMD5(userID) -// msg := &sdkws.MsgData{ -// SendID: userID, -// ClientMsgID: msgID, -// ServerMsgID: msgID, -// SenderPlatformID: constant.AdminPlatformID, -// SessionType: constant.NotificationChatType, -// ContentType: constant.UserSubscribeOnlineStatusNotification, -// Content: msgContent, -// SendTime: now, -// CreateTime: now, -// } -// for _, client := range clients { -// msg.RecvID = client.UserID -// if err := client.PushMessage(ctx, msg); err != nil { -// log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) -// } -// } -//} +func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { + if len(addUserIDs)+len(delUserIDs) == 0 { + return + } + var ( + del = make(map[string]struct{}) + add = make(map[string]struct{}) + ) + client.subLock.Lock() + for _, userID := range delUserIDs { + if _, ok := client.subUserIDs[userID]; !ok { + continue + } + del[userID] = struct{}{} + delete(client.subUserIDs, userID) + } + for _, userID := range addUserIDs { + delete(del, userID) + if _, ok := client.subUserIDs[userID]; ok { + continue + } + client.subUserIDs[userID] = struct{}{} + } + client.subLock.Unlock() + if len(del)+len(add) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for userID := range del { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } + for userID := range add { + sub, ok := s.userIDs[userID] + if !ok { + sub = &subClient{clients: make(map[string]*Client)} + s.userIDs[userID] = sub + } + sub.clients[addr] = client + } +} + +func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { + clients := ws.subscription.GetClient(userID) + if len(clients) == 0 { + return + } + msgContent, err := json.Marshal(platformIDs) + if err != nil { + log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) + return + } + now := time.Now().UnixMilli() + msgID := idutil.GetMsgIDByMD5(userID) + msg := &sdkws.MsgData{ + SendID: userID, + ClientMsgID: msgID, + ServerMsgID: msgID, + SenderPlatformID: constant.AdminPlatformID, + SessionType: constant.NotificationChatType, + ContentType: constant.UserSubscribeOnlineStatusNotification, + Content: msgContent, + SendTime: now, + CreateTime: now, + } + for _, client := range clients { + msg.RecvID = client.UserID + if err := client.PushMessage(ctx, msg); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) + } + } +} diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index e942f40ae9..ed87b3929f 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -75,7 +75,7 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config - consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb) + consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) return &consumerHandler, nil } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index d986651c7d..5db68d198d 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -15,7 +15,7 @@ import ( "time" ) -func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient) *OnlineCache { +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) *OnlineCache { x := &OnlineCache{ user: user, group: group, @@ -33,6 +33,9 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re } storageCache := x.setUserOnline(userID, platformIDs) log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + if fn != nil { + fn(ctx, userID, platformIDs) + } } }() return x From e17f0c95aa2e5702a32237de348363fc50ecaf9d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 14:44:26 +0800 Subject: [PATCH 108/147] conversation version incremental --- go.mod | 2 +- go.sum | 4 +- internal/api/conversation.go | 8 +++ internal/api/router.go | 2 + internal/rpc/conversation/conversaion.go | 14 +++- internal/rpc/conversation/sync.go | 56 ++++++++++++++++ internal/rpc/third/s3.go | 5 ++ internal/rpc/third/third.go | 1 + internal/rpc/user/user.go | 5 ++ .../storage/cache/cachekey/conversation.go | 5 ++ pkg/common/storage/cache/conversation.go | 4 ++ .../storage/cache/redis/conversation.go | 17 +++++ pkg/common/storage/controller/conversation.go | 17 ++++- pkg/common/storage/database/conversation.go | 2 +- .../storage/database/mgo/conversation.go | 65 ++++++++++++++----- pkg/common/storage/database/name.go | 27 ++++---- 16 files changed, 198 insertions(+), 36 deletions(-) create mode 100644 internal/rpc/conversation/sync.go diff --git a/go.mod b/go.mod index 2111208f29..9b3a0dfca5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/protocol v0.0.69-alpha.27 github.com/openimsdk/tools v0.0.49-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5bc9f6d984..2463159574 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.27 h1:0Ctpu9VBXVCkKno6vVNBgUTyo9W9bG7SZuAhQr/4H8Y= +github.com/openimsdk/protocol v0.0.69-alpha.27/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/conversation.go b/internal/api/conversation.go index f273eaa4a7..cb514bb7da 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -50,3 +50,11 @@ func (o *ConversationApi) SetConversations(c *gin.Context) { func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { a2r.Call(conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client, c) } + +func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client, c) +} + +func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetIncrementalConversation, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 0f46f26baf..76a7eeea07 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -192,6 +192,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/get_conversations", c.GetConversations) conversationGroup.POST("/set_conversations", c.SetConversations) conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) + conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) + conversationGroup.POST("/get_incremental_conversation", c.GetIncrementalConversation) } statisticsGroup := r.Group("/statistics") diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index df9267ae04..9571ed7b27 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -184,13 +184,23 @@ func (c *conversationServer) GetAllConversations(ctx context.Context, req *pbcon } func (c *conversationServer) GetConversations(ctx context.Context, req *pbconversation.GetConversationsReq) (*pbconversation.GetConversationsResp, error) { - conversations, err := c.conversationDatabase.FindConversations(ctx, req.OwnerUserID, req.ConversationIDs) + conversations, err := c.getConversations(ctx, req.OwnerUserID, req.ConversationIDs) + if err != nil { + return nil, err + } + return &pbconversation.GetConversationsResp{ + Conversations: conversations, + }, nil +} + +func (c *conversationServer) getConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { + conversations, err := c.conversationDatabase.FindConversations(ctx, ownerUserID, conversationIDs) if err != nil { return nil, err } resp := &pbconversation.GetConversationsResp{Conversations: []*pbconversation.Conversation{}} resp.Conversations = convert.ConversationsDB2Pb(conversations) - return resp, nil + return convert.ConversationsDB2Pb(conversations), nil } func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go new file mode 100644 index 0000000000..29c11c4a10 --- /dev/null +++ b/internal/rpc/conversation/sync.go @@ -0,0 +1,56 @@ +package conversation + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/conversation" +) + +func (c *conversationServer) GetFullOwnerConversationIDs(ctx context.Context, req *conversation.GetFullOwnerConversationIDsReq) (*conversation.GetFullOwnerConversationIDsResp, error) { + vl, err := c.conversationDatabase.FindMaxConversationUserVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(conversationIDs) + if req.IdHash == idHash { + conversationIDs = nil + } + return &conversation.GetFullOwnerConversationIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + ConversationIDs: conversationIDs, + }, nil +} + +func (c *conversationServer) GetIncrementalConversation(ctx context.Context, req *conversation.GetIncrementalConversationReq) (*conversation.GetIncrementalConversationResp, error) { + opt := incrversion.Option[*conversation.Conversation, conversation.GetIncrementalConversationResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: c.conversationDatabase.FindConversationUserVersion, + CacheMaxVersion: c.conversationDatabase.FindMaxConversationUserVersionCache, + Find: func(ctx context.Context, conversationIDs []string) ([]*conversation.Conversation, error) { + return c.getConversations(ctx, req.UserID, conversationIDs) + }, + ID: func(elem *conversation.Conversation) string { return elem.GroupID }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*conversation.Conversation, full bool) *conversation.GetIncrementalConversationResp { + return &conversation.GetIncrementalConversationResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 4cb1b81d06..881a5a3328 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -290,3 +290,8 @@ type FormDataMate struct { Group string `json:"group"` Key string `json:"key"` } + +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 7560486a00..d0d88b174b 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -43,6 +43,7 @@ type thirdServer struct { defaultExpire time.Duration config *Config } + type Config struct { RpcConfig config.Third RedisConfig config.Redis diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7b..144a37602b 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -736,3 +736,8 @@ func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (* } return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/pkg/common/storage/cache/cachekey/conversation.go b/pkg/common/storage/cache/cachekey/conversation.go index aea4ceec68..d19fcc5767 100644 --- a/pkg/common/storage/cache/cachekey/conversation.go +++ b/pkg/common/storage/cache/cachekey/conversation.go @@ -23,6 +23,7 @@ const ( SuperGroupRecvMsgNotNotifyUserIDsKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:" SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:" ConversationNotReceiveMessageUserIDsKey = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:" + ConversationUserMaxKey = "CONVERSATION_USER_MAX:" ) func GetConversationKey(ownerUserID, conversationID string) string { @@ -56,3 +57,7 @@ func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string { func GetUserConversationIDsHashKey(ownerUserID string) string { return ConversationIDsHashKey + ownerUserID } + +func GetConversationUserMaxVersionKey(userID string) string { + return ConversationUserMaxKey + userID +} diff --git a/pkg/common/storage/cache/conversation.go b/pkg/common/storage/cache/conversation.go index f34fd599f8..bc17614836 100644 --- a/pkg/common/storage/cache/conversation.go +++ b/pkg/common/storage/cache/conversation.go @@ -54,4 +54,8 @@ type ConversationCache interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache + + DelConversationVersionUserIDs(userIDs ...string) ConversationCache + + FindMaxConversationUserVersion(ctx context.Context, userID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 8c0393dd56..c491d1b947 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -95,6 +95,10 @@ func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID strin return cachekey.GetUserConversationIDsHashKey(ownerUserID) } +func (c *ConversationRedisCache) getConversationUserMaxVersionKey(ownerUserID string) string { + return cachekey.GetConversationUserMaxVersionKey(ownerUserID) +} + func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { return getCache(ctx, c.rcClient, c.getConversationIDsKey(ownerUserID), c.expireTime, func(ctx context.Context) ([]string, error) { return c.conversationDB.FindUserIDAllConversationID(ctx, ownerUserID) @@ -233,6 +237,19 @@ func (c *ConversationRedisCache) DelConversationNotReceiveMessageUserIDs(convers for _, conversationID := range conversationIDs { cache.AddKeys(c.getConversationNotReceiveMessageUserIDsKey(conversationID)) } + return cache +} +func (c *ConversationRedisCache) DelConversationVersionUserIDs(userIDs ...string) cache.ConversationCache { + cache := c.CloneConversationCache() + for _, userID := range userIDs { + cache.AddKeys(c.getConversationUserMaxVersionKey(userID)) + } return cache } + +func (c *ConversationRedisCache) FindMaxConversationUserVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, c.rcClient, c.getConversationUserMaxVersionKey(userID), c.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index 18ef3f8bad..d4b39fb8b9 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -66,6 +66,8 @@ type ConversationDatabase interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) // GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) // FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) + FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -106,6 +108,7 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, if _, ok := fieldMap["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) } + cache = cache.DelConversationVersionUserIDs(haveUserIDs...) } NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs) log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs) @@ -137,7 +140,7 @@ func (c *conversationDatabase) UpdateUsersConversationField(ctx context.Context, return err } cache := c.cache.CloneConversationCache() - cache = cache.DelUsersConversation(conversationID, userIDs...) + cache = cache.DelUsersConversation(conversationID, userIDs...).DelConversationVersionUserIDs(userIDs...) if _, ok := args["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversationID) } @@ -155,13 +158,14 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) userIDs = append(userIDs, conversation.OwnerUserID) } - return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ChainExecDel(ctx) + return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).DelConversationVersionUserIDs(userIDs...).ChainExecDel(ctx) } func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Context, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() for _, conversation := range conversations { + cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID) for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} { ownerUserID := v[0] userID := v[1] @@ -207,6 +211,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() + cache = cache.DelConversationVersionUserIDs(ownerUserID) groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.Conversation) (string, bool) { return e.GroupID, e.GroupID != "" })) @@ -322,3 +327,11 @@ func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Contex func (c *conversationDatabase) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { return c.cache.GetConversationNotReceiveMessageUserIDs(ctx, conversationID) } + +func (c *conversationDatabase) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, version, limit) +} + +func (c *conversationDatabase) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) { + return c.cache.FindMaxConversationUserVersion(ctx, userID) +} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 46aa02d98e..85f3dd668a 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -22,7 +22,6 @@ import ( type Conversation interface { Create(ctx context.Context, conversations []*model.Conversation) (err error) - Delete(ctx context.Context, groupIDs []string) (err error) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) Update(ctx context.Context, conversation *model.Conversation) (err error) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) @@ -39,4 +38,5 @@ type Conversation interface { GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index b462d39583..3d505f1d34 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -41,40 +41,71 @@ func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { if err != nil { return nil, errs.Wrap(err) } - return &ConversationMgo{coll: coll}, nil + version, err := NewVersionLog(db.Collection(database.ConversationVersionName)) + if err != nil { + return nil, err + } + return &ConversationMgo{version: version, coll: coll}, nil } type ConversationMgo struct { - coll *mongo.Collection + version database.VersionLog + coll *mongo.Collection } func (c *ConversationMgo) Create(ctx context.Context, conversations []*model.Conversation) (err error) { - return mongoutil.InsertMany(ctx, c.coll, conversations) -} - -func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err error) { - return mongoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, c.coll, conversations) + }, func() error { + userConversation := make(map[string][]string) + for _, conversation := range conversations { + userConversation[conversation.OwnerUserID] = append(userConversation[conversation.OwnerUserID], conversation.ConversationID) + } + for userID, conversationIDs := range userConversation { + if err := c.version.IncrVersion(ctx, userID, conversationIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } -func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) { - if len(args) == 0 { +func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (int64, error) { + if len(args) == 0 || len(userIDs) == 0 { return 0, nil } filter := bson.M{ "conversation_id": conversationID, + "owner_user_id": bson.M{"$in": userIDs}, } - if len(userIDs) > 0 { - filter["owner_user_id"] = bson.M{"$in": userIDs} - } - res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + var rows int64 + err := mongoutil.IncrVersion(func() error { + res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + if err != nil { + return err + } + rows = res.ModifiedCount + return nil + }, func() error { + for _, userID := range userIDs { + if err := c.version.IncrVersion(ctx, userID, []string{conversationID}, model.VersionStateUpdate); err != nil { + return err + } + } + return nil + }) if err != nil { return 0, err } - return res.ModifiedCount, nil + return rows, nil } func (c *ConversationMgo) Update(ctx context.Context, conversation *model.Conversation) (err error) { - return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + }, func() error { + return c.version.IncrVersion(ctx, conversation.OwnerUserID, []string{conversation.ConversationID}, model.VersionStateUpdate) + }) } func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) { @@ -178,3 +209,7 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}), ) } + +func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return c.version.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 986f22a1a2..3d9eb6a977 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -1,17 +1,18 @@ package database const ( - BlackName = "black" - ConversationName = "conversation" - FriendName = "friend" - FriendVersionName = "friend_version" - FriendRequestName = "friend_request" - GroupName = "group" - GroupMemberName = "group_member" - GroupMemberVersionName = "group_member_version" - GroupJoinVersionName = "group_join_version" - GroupRequestName = "group_request" - LogName = "log" - ObjectName = "s3" - UserName = "user" + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + ConversationVersionName = "conversation_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" ) From 92d70ece32026faf7d737d8b2952b223435942fd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 15:18:39 +0800 Subject: [PATCH 109/147] merge seq --- go.sum | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/go.sum b/go.sum index f452bcd871..0be33db42f 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -222,12 +220,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= -github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= @@ -272,8 +264,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= -github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= +github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= +github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -527,6 +519,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From c5519ed6c9e5c1a380bb40ea1e43c7f4c29e9940 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 15:31:57 +0800 Subject: [PATCH 110/147] merge online --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 92350d7457..577154a1b6 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From c3729d1e99d7c2c5d255668f02a8fb3c7bae9195 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 15:38:54 +0800 Subject: [PATCH 111/147] merge online --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6450f36158..52a82010b7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.24 + github.com/openimsdk/protocol v0.0.69-alpha.27 github.com/openimsdk/tools v0.0.49-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 577154a1b6..2463159574 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= -github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.27 h1:0Ctpu9VBXVCkKno6vVNBgUTyo9W9bG7SZuAhQr/4H8Y= +github.com/openimsdk/protocol v0.0.69-alpha.27/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From e1e3bb82489604678d4cdc70cc8e1419c3effa68 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 15:40:41 +0800 Subject: [PATCH 112/147] merge online --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 577154a1b6..2463159574 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= -github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.27 h1:0Ctpu9VBXVCkKno6vVNBgUTyo9W9bG7SZuAhQr/4H8Y= +github.com/openimsdk/protocol v0.0.69-alpha.27/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 1d54c61b8cb9339cfac4551f89f40cdb0e7f8df7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 10 Jul 2024 15:42:52 +0800 Subject: [PATCH 113/147] merge seq --- internal/rpc/third/s3.go | 5 ----- internal/rpc/user/user.go | 5 ----- 2 files changed, 10 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index f70a6d142f..54cbc30c1a 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -340,8 +340,3 @@ type FormDataMate struct { Group string `json:"group"` Key string `json:"key"` } - -func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - //TODO implement me - panic("implement me") -} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 75d01d2615..0b96077ec3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -669,8 +669,3 @@ func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (* } return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } - -func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { - //TODO implement me - panic("implement me") -} From ec7bebfb8656accbb3246081999f39ad6a976e48 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 11 Jul 2024 10:52:18 +0800 Subject: [PATCH 114/147] GetOwnerConversation --- go.mod | 2 +- go.sum | 4 ++-- internal/api/conversation.go | 4 ++++ internal/api/router.go | 1 + internal/rpc/conversation/conversaion.go | 11 +++++++++++ pkg/common/storage/controller/conversation.go | 18 ++++++++++++++++++ 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9b3a0dfca5..5ea0a62cce 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.27 + github.com/openimsdk/protocol v0.0.69-alpha.30 github.com/openimsdk/tools v0.0.49-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 2463159574..dd6069abe0 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.27 h1:0Ctpu9VBXVCkKno6vVNBgUTyo9W9bG7SZuAhQr/4H8Y= -github.com/openimsdk/protocol v0.0.69-alpha.27/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= +github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/conversation.go b/internal/api/conversation.go index cb514bb7da..360313ea87 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -58,3 +58,7 @@ func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { a2r.Call(conversation.ConversationClient.GetIncrementalConversation, o.Client, c) } + +func (o *ConversationApi) GetOwnerConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetOwnerConversation, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 76a7eeea07..2e7d0d7a2d 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -194,6 +194,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) conversationGroup.POST("/get_incremental_conversation", c.GetIncrementalConversation) + conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) } statisticsGroup := r.Group("/statistics") diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 9571ed7b27..3047c376bb 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -591,3 +591,14 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv } return &pbconversation.UpdateConversationResp{}, nil } + +func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbconversation.GetOwnerConversationReq) (*pbconversation.GetOwnerConversationResp, error) { + total, conversations, err := c.conversationDatabase.GetOwnerConversation(ctx, req.UserID, req.Pagination) + if err != nil { + return nil, err + } + return &pbconversation.GetOwnerConversationResp{ + Total: total, + Conversations: convert.ConversationsDB2Pb(conversations), + }, nil +} diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index d4b39fb8b9..c804d1cc51 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -68,6 +68,7 @@ type ConversationDatabase interface { // FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) + GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -335,3 +336,20 @@ func (c *conversationDatabase) FindConversationUserVersion(ctx context.Context, func (c *conversationDatabase) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) { return c.cache.FindMaxConversationUserVersion(ctx, userID) } + +func (c *conversationDatabase) GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) { + conversationIDs, err := c.cache.GetUserConversationIDs(ctx, ownerUserID) + if err != nil { + return 0, nil, err + } + findConversationIDs := datautil.Paginate(conversationIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) + conversations := make([]*relationtb.Conversation, 0, len(findConversationIDs)) + for _, conversationID := range findConversationIDs { + conversation, err := c.cache.GetConversation(ctx, ownerUserID, conversationID) + if err != nil { + return 0, nil, err + } + conversations = append(conversations, conversation) + } + return int64(len(conversationIDs)), conversations, nil +} From 29837c95a11da67f0c9b662d3678e505d877d7af Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Thu, 11 Jul 2024 18:17:50 +0800 Subject: [PATCH 115/147] fix: change incremental syncer router name. --- internal/api/friend.go | 3 +++ internal/api/router.go | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/api/friend.go b/internal/api/friend.go index 11d7375fa3..6c42184230 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -16,6 +16,7 @@ package api import ( "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" @@ -100,6 +101,8 @@ func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) } +// GetIncrementalBlacks is temporarily unused. +// Deprecated: This function is currently unused and may be removed in future versions. func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 2e7d0d7a2d..822e647932 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -8,6 +8,9 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -15,8 +18,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { @@ -118,9 +119,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) - groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) - groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_incremental_join_groups", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_members", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_members_batch", g.GetIncrementalGroupMemberBatch) groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } @@ -193,7 +194,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/set_conversations", c.SetConversations) conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) - conversationGroup.POST("/get_incremental_conversation", c.GetIncrementalConversation) + conversationGroup.POST("/get_incremental_conversations", c.GetIncrementalConversation) conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) } From 326dc3836cd969bceddc1d56aeebb607aa5f2204 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 11 Jul 2024 18:47:25 +0800 Subject: [PATCH 116/147] rockscache batch get --- pkg/common/storage/cache/group.go | 1 - pkg/common/storage/cache/redis/batch.go | 87 +++++++++++++++++ .../storage/cache/redis/batch_handler.go | 55 ++++++----- pkg/common/storage/cache/redis/batch_test.go | 79 +++++++++++++++ .../storage/cache/redis/conversation.go | 8 +- pkg/common/storage/cache/redis/friend.go | 25 ----- pkg/common/storage/cache/redis/group.go | 90 +++-------------- pkg/common/storage/cache/redis/msg.go | 1 - pkg/common/storage/cache/redis/user.go | 96 +------------------ pkg/common/storage/controller/user.go | 2 +- pkg/common/storage/database/group_member.go | 2 + .../storage/database/mgo/group_member.go | 16 ++++ 12 files changed, 236 insertions(+), 226 deletions(-) create mode 100644 pkg/common/storage/cache/redis/batch.go create mode 100644 pkg/common/storage/cache/redis/batch_test.go diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 73479bb1bb..91953d9f9a 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -36,7 +36,6 @@ type GroupCache interface { DelGroupMembersHash(groupID string) GroupCache GetGroupMemberIDs(ctx context.Context, groupID string) (groupMemberIDs []string, err error) - GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (groupMemberIDs map[string][]string, err error) DelGroupMemberIDs(groupID string) GroupCache diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go new file mode 100644 index 0000000000..8434474cb4 --- /dev/null +++ b/pkg/common/storage/cache/redis/batch.go @@ -0,0 +1,87 @@ +package redis + +import ( + "context" + "encoding/json" + "github.com/dtm-labs/rockscache" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + "time" + "unsafe" +) + +func getRocksCacheRedisClient(cli *rockscache.Client) redis.UniversalClient { + type Client struct { + rdb redis.UniversalClient + _ rockscache.Options + _ singleflight.Group + } + return (*Client)(unsafe.Pointer(cli)).rdb +} + +func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v V) K, fn func(ctx context.Context, ids []K) ([]V, error)) ([]V, error) { + if len(ids) == 0 { + return nil, nil + } + findKeys := make([]string, 0, len(ids)) + keyId := make(map[string]K) + for _, id := range ids { + key := idKey(id) + if _, ok := keyId[key]; ok { + continue + } + keyId[key] = id + findKeys = append(findKeys, key) + } + slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), findKeys) + if err != nil { + return nil, err + } + result := make([]V, 0, len(findKeys)) + for _, keys := range slotKeys { + indexCache, err := rcClient.FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { + queryIds := make([]K, 0, len(idx)) + idIndex := make(map[K]int) + for _, index := range idx { + id := keyId[keys[index]] + idIndex[id] = index + queryIds = append(queryIds, id) + } + values, err := fn(ctx, queryIds) + if err != nil { + return nil, err + } + if len(values) == 0 { + return map[int]string{}, nil + } + cacheIndex := make(map[int]string) + for _, value := range values { + id := vId(value) + index, ok := idIndex[id] + if !ok { + continue + } + bs, err := json.Marshal(value) + if err != nil { + return nil, err + } + cacheIndex[index] = string(bs) + } + return cacheIndex, nil + }) + if err != nil { + return nil, err + } + for _, data := range indexCache { + if data == "" { + continue + } + var value V + if err := json.Unmarshal([]byte(data), &value); err != nil { + return nil, err + } + result = append(result, value) + } + } + return result, nil +} diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 95f6699047..52e046a408 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -23,7 +23,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw/specialerror" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" @@ -147,30 +146,30 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin return t, nil } -func batchGetCache[T any, K comparable]( - ctx context.Context, - rcClient *rockscache.Client, - expire time.Duration, - keys []K, - keyFn func(key K) string, - fns func(ctx context.Context, key K) (T, error), -) ([]T, error) { - if len(keys) == 0 { - return nil, nil - } - res := make([]T, 0, len(keys)) - for _, key := range keys { - val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { - return fns(ctx, key) - }) - if err != nil { - if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { - continue - } - return nil, errs.Wrap(err) - } - res = append(res, val) - } - - return res, nil -} +//func batchGetCache[T any, K comparable]( +// ctx context.Context, +// rcClient *rockscache.Client, +// expire time.Duration, +// keys []K, +// keyFn func(key K) string, +// fns func(ctx context.Context, key K) (T, error), +//) ([]T, error) { +// if len(keys) == 0 { +// return nil, nil +// } +// res := make([]T, 0, len(keys)) +// for _, key := range keys { +// val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { +// return fns(ctx, key) +// }) +// if err != nil { +// if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { +// continue +// } +// return nil, errs.Wrap(err) +// } +// res = append(res, val) +// } +// +// return res, nil +//} diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go new file mode 100644 index 0000000000..b9174abbf6 --- /dev/null +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -0,0 +1,79 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "testing" + "time" +) + +func TestName(t *testing.T) { + //var rocks rockscache.Client + //rdb := getRocksCacheRedisClient(&rocks) + //t.Log(rdb == nil) + + ctx := context.Background() + rdb, err := redisutil.NewRedisClient(ctx, (&config.Redis{ + Address: []string{"172.16.8.48:16379"}, + Password: "openIM123", + DB: 3, + }).Build()) + if err != nil { + panic(err) + } + mgocli, err := mongoutil.NewMongoDB(ctx, (&config.Mongo{ + Address: []string{"172.16.8.48:37017"}, + Database: "openim_v3", + Username: "openIM", + Password: "openIM123", + MaxPoolSize: 100, + MaxRetry: 1, + }).Build()) + if err != nil { + panic(err) + } + userMgo, err := mgo.NewUserMongo(mgocli.GetDB()) + if err != nil { + panic(err) + } + rock := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) + //var keys []string + //for i := 1; i <= 10; i++ { + // keys = append(keys, fmt.Sprintf("test%d", i)) + //} + //res, err := cli.FetchBatch2(ctx, keys, time.Hour, func(idx []int) (map[int]string, error) { + // t.Log("FetchBatch2=>", idx) + // time.Sleep(time.Second * 1) + // res := make(map[int]string) + // for _, i := range idx { + // res[i] = fmt.Sprintf("hello_%d", i) + // } + // t.Log("FetchBatch2=>", res) + // return res, nil + //}) + //if err != nil { + // t.Log(err) + // return + //} + //t.Log(res) + + userIDs := []string{"1814217053", "2110910952", "1234567890"} + + res, err := batchGetCache2(ctx, rock, time.Hour, userIDs, func(id string) string { + return "TEST_USER:" + id + }, func(v *model.User) string { + return v.UserID + }, func(ctx context.Context, ids []string) ([]*model.User, error) { + t.Log("find mongo", ids) + return userMgo.Find(ctx, ids) + }) + if err != nil { + panic(err) + } + t.Log("==>", res) +} diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index c491d1b947..95e680afb4 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -164,10 +164,12 @@ func (c *ConversationRedisCache) DelConversations(ownerUserID string, conversati } func (c *ConversationRedisCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*model.Conversation, error) { - return batchGetCache(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { + return batchGetCache2(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { return c.getConversationKey(ownerUserID, conversationID) - }, func(ctx context.Context, conversationID string) (*model.Conversation, error) { - return c.conversationDB.Take(ctx, ownerUserID, conversationID) + }, func(conversation *model.Conversation) string { + return conversation.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) { + return c.conversationDB.Find(ctx, ownerUserID, conversationIDs) }) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 01988310c9..be4687794f 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -70,10 +70,6 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } -//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { -// return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) -//} - func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { return cachekey.GetFriendMaxVersionKey(ownerUserID) } @@ -107,16 +103,6 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { -// newGroupCache := f.CloneFriendCache() -// keys := make([]string, 0, len(ownerUserIDs)) -// for _, userID := range ownerUserIDs { -// keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) -// } -// newGroupCache.AddKeys(keys...) -// return newGroupCache -//} - // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { friendIDs, err := f.GetFriendIDs(ctx, ownerUserID) @@ -193,17 +179,6 @@ func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.Fri return newFriendCache } -//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { -// userIDs, err := f.GetFriendIDs(ctx, ownerUserID) -// if err != nil { -// return nil, err -// } -// if len(userIDs) > f.syncCount { -// userIDs = userIDs[:f.syncCount] -// } -// return userIDs, nil -//} - func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 589678c506..d327c218f3 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -118,34 +118,12 @@ func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { return cachekey.GetJoinGroupMaxVersionKey(userID) } -func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { - key := g.getGroupInfoKey(group.GroupID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } - - return 0, errIndex -} - -func (g *GroupCacheRedis) GetGroupMemberIndex(groupMember *model.GroupMember, keys []string) (int, error) { - key := g.getGroupMemberInfoKey(groupMember.GroupID, groupMember.UserID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } - - return 0, errIndex +func (g *GroupCacheRedis) getGroupID(group *model.Group) string { + return group.GroupID } func (g *GroupCacheRedis) GetGroupsInfo(ctx context.Context, groupIDs []string) (groups []*model.Group, err error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { - return g.getGroupInfoKey(groupID) - }, func(ctx context.Context, groupID string) (*model.Group, error) { - return g.groupDB.Take(ctx, groupID) - }) + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, g.getGroupInfoKey, g.getGroupID, g.groupDB.Find) } func (g *GroupCacheRedis) GetGroupInfo(ctx context.Context, groupID string) (group *model.Group, err error) { @@ -233,19 +211,6 @@ func (g *GroupCacheRedis) GetGroupMemberIDs(ctx context.Context, groupID string) }) } -func (g *GroupCacheRedis) GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (map[string][]string, error) { - m := make(map[string][]string) - for _, groupID := range groupIDs { - userIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - m[groupID] = userIDs - } - - return m, nil -} - func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { cache := g.CloneGroupCache() cache.AddKeys(g.getGroupMemberIDsKey(groupID)) @@ -285,10 +250,12 @@ func (g *GroupCacheRedis) GetGroupMemberInfo(ctx context.Context, groupID, userI } func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, userID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.UserID + }, func(ctx context.Context, userIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.Find(ctx, groupID, userIDs) }) } @@ -301,14 +268,6 @@ func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID st return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) } -func (g *GroupCacheRedis) GetAllGroupMemberInfo(ctx context.Context, groupID string) ([]*model.GroupMember, error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) -} - func (g *GroupCacheRedis) DelGroupMembersInfo(groupID string, userIDs ...string) cache.GroupCache { keys := make([]string, 0, len(userIDs)) for _, userID := range userIDs { @@ -388,42 +347,23 @@ func (g *GroupCacheRedis) GetGroupRolesLevelMemberInfo(ctx context.Context, grou return g.GetGroupMembersInfo(ctx, groupID, userIDs) } -func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) (_ []*model.GroupMember, err error) { +func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) { if len(groupIDs) == 0 { + var err error groupIDs, err = g.GetJoinedGroupIDs(ctx, userID) if err != nil { return nil, err } } - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, groupID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.GroupID + }, func(ctx context.Context, groupIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.FindInGroup(ctx, userID, groupIDs) }) } -//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { -// userIDs, err := g.GetGroupMemberIDs(ctx, groupID) -// if err != nil { -// return nil, err -// } -// if len(userIDs) > g.syncCount { -// userIDs = userIDs[:g.syncCount] -// } -// return userIDs, nil -//} -// -//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { -// groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) -// if err != nil { -// return nil, err -// } -// if len(groupIDs) > g.syncCount { -// groupIDs = groupIDs[:g.syncCount] -// } -// return groupIDs, nil -//} - func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { keys := make([]string, 0, len(groupIDs)) for _, groupID := range groupIDs { diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 2d21cfe135..30f367bb75 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -183,5 +183,4 @@ func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, return nil, nil, err } return seqMsgs, failedSeqs, nil - } diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index c3accd2c32..f6b4907302 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -16,16 +16,12 @@ package redis import ( "context" - "encoding/json" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "time" @@ -58,9 +54,9 @@ func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, } } -//func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { -// return cachekey.GetOnlineStatusKey(modKey) -//} +func (u *UserCacheRedis) getUserID(user *model.User) string { + return user.UserID +} func (u *UserCacheRedis) CloneUserCache() cache.UserCache { return &UserCacheRedis{ @@ -87,11 +83,7 @@ func (u *UserCacheRedis) GetUserInfo(ctx context.Context, userID string) (userIn } func (u *UserCacheRedis) GetUsersInfo(ctx context.Context, userIDs []string) ([]*model.User, error) { - return batchGetCache(ctx, u.rcClient, u.expireTime, userIDs, func(userID string) string { - return u.getUserInfoKey(userID) - }, func(ctx context.Context, userID string) (*model.User, error) { - return u.userDB.Take(ctx, userID) - }) + return batchGetCache2(ctx, u.rcClient, u.expireTime, userIDs, u.getUserInfoKey, u.getUserID, u.userDB.Find) } func (u *UserCacheRedis) DelUsersInfo(userIDs ...string) cache.UserCache { @@ -127,83 +119,3 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } - -func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { - if isNil { - log.ZWarn(ctx, "this user not online,maybe trigger order not right", - err, "userStatus", status) - - return nil - } - var onlineStatus user.OnlineStatus - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - var newPlatformIDs []int32 - for _, val := range onlineStatus.PlatformIDs { - if val != platformID { - newPlatformIDs = append(newPlatformIDs, val) - } - } - if newPlatformIDs == nil { - _, err = u.rdb.HDel(ctx, key, userID).Result() - if err != nil { - return errs.Wrap(err) - } - } else { - onlineStatus.PlatformIDs = newPlatformIDs - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} - -func (u *UserCacheRedis) refreshStatusOnline(ctx context.Context, userID string, platformID int32, isNil bool, err error, result, key string) error { - var onlineStatus user.OnlineStatus - if !isNil { - err := json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - onlineStatus.PlatformIDs = RemoveRepeatedElementsInList(append(onlineStatus.PlatformIDs, platformID)) - } else { - onlineStatus.PlatformIDs = append(onlineStatus.PlatformIDs, platformID) - } - onlineStatus.Status = constant.Online - onlineStatus.UserID = userID - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.WrapMsg(err, "json.Marshal failed") - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - - return nil -} - -type Comparable interface { - ~int | ~string | ~float64 | ~int32 -} - -func RemoveRepeatedElementsInList[T Comparable](slc []T) []T { - var result []T - tempMap := map[T]struct{}{} - for _, e := range slc { - if _, found := tempMap[e]; !found { - tempMap[e] = struct{}{} - result = append(result, e) - } - } - - return result -} diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 321eff03cd..59559537b1 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -195,7 +195,7 @@ func (u *userDatabase) GetAllUserID(ctx context.Context, pagination pagination.P } func (u *userDatabase) GetUserByID(ctx context.Context, userID string) (user *model.User, err error) { - return u.userDB.Take(ctx, userID) + return u.cache.GetUserInfo(ctx, userID) } // CountTotal Get the total number of users. diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index c272b6ef6f..43a7e6095e 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -28,6 +28,8 @@ type GroupMember interface { UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) + Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) + FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 3eb93a10ec..f89822d3c4 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -153,6 +153,22 @@ func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) ( return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } +func (g *GroupMemberMgo) Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"group_id": groupID} + if len(userIDs) > 0 { + filter["user_id"] = bson.M{"$in": userIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) +} + +func (g *GroupMemberMgo) FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"user_id": userID} + if len(groupIDs) > 0 { + filter["group_id"] = bson.M{"$in": groupIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) +} + func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { return mongoutil.FindOne[*model.GroupMember](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}) } From a7d1a5351c21e6cdbc29a6e5f1eec0c29bc7e7ed Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 12 Jul 2024 18:44:20 +0800 Subject: [PATCH 117/147] rockscache seq batch get --- pkg/common/storage/cache/redis/batch.go | 15 ++- pkg/common/storage/cache/redis/batch_test.go | 46 ++------ .../cache/redis/redis_shard_manager.go | 16 ++- .../storage/cache/redis/seq_conversation.go | 92 ++++++++++++++-- pkg/common/storage/cache/redis/seq_user.go | 104 +++++++++++++++++- pkg/common/storage/cache/seq_conversation.go | 2 + pkg/common/storage/cache/seq_user.go | 3 + pkg/common/storage/controller/msg.go | 79 +------------ pkg/common/storage/database/mgo/seq_user.go | 18 +++ pkg/common/storage/database/seq_user.go | 1 + 10 files changed, 251 insertions(+), 125 deletions(-) diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 8434474cb4..5f9a8c82db 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -19,7 +19,7 @@ func getRocksCacheRedisClient(cli *rockscache.Client) redis.UniversalClient { return (*Client)(unsafe.Pointer(cli)).rdb } -func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v V) K, fn func(ctx context.Context, ids []K) ([]V, error)) ([]V, error) { +func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v *V) K, fn func(ctx context.Context, ids []K) ([]*V, error)) ([]*V, error) { if len(ids) == 0 { return nil, nil } @@ -37,7 +37,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac if err != nil { return nil, err } - result := make([]V, 0, len(findKeys)) + result := make([]*V, 0, len(findKeys)) for _, keys := range slotKeys { indexCache, err := rcClient.FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { queryIds := make([]K, 0, len(idx)) @@ -72,7 +72,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac if err != nil { return nil, err } - for _, data := range indexCache { + for index, data := range indexCache { if data == "" { continue } @@ -80,8 +80,15 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac if err := json.Unmarshal([]byte(data), &value); err != nil { return nil, err } - result = append(result, value) + if cb, ok := any(&value).(BatchCacheCallback[K]); ok { + cb.BatchCache(keyId[keys[index]]) + } + result = append(result, &value) } } return result, nil } + +type BatchCacheCallback[K comparable] interface { + BatchCache(id K) +} diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go index b9174abbf6..e4caa2a21f 100644 --- a/pkg/common/storage/cache/redis/batch_test.go +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -2,14 +2,11 @@ package redis import ( "context" - "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "testing" - "time" ) func TestName(t *testing.T) { @@ -37,43 +34,22 @@ func TestName(t *testing.T) { if err != nil { panic(err) } - userMgo, err := mgo.NewUserMongo(mgocli.GetDB()) + //userMgo, err := mgo.NewUserMongo(mgocli.GetDB()) + //if err != nil { + // panic(err) + //} + //rock := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) + mgoSeqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) if err != nil { panic(err) } - rock := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) - //var keys []string - //for i := 1; i <= 10; i++ { - // keys = append(keys, fmt.Sprintf("test%d", i)) - //} - //res, err := cli.FetchBatch2(ctx, keys, time.Hour, func(idx []int) (map[int]string, error) { - // t.Log("FetchBatch2=>", idx) - // time.Sleep(time.Second * 1) - // res := make(map[int]string) - // for _, i := range idx { - // res[i] = fmt.Sprintf("hello_%d", i) - // } - // t.Log("FetchBatch2=>", res) - // return res, nil - //}) - //if err != nil { - // t.Log(err) - // return - //} - //t.Log(res) - - userIDs := []string{"1814217053", "2110910952", "1234567890"} + seqUser := NewSeqUserCacheRedis(rdb, mgoSeqUser) - res, err := batchGetCache2(ctx, rock, time.Hour, userIDs, func(id string) string { - return "TEST_USER:" + id - }, func(v *model.User) string { - return v.UserID - }, func(ctx context.Context, ids []string) ([]*model.User, error) { - t.Log("find mongo", ids) - return userMgo.Find(ctx, ids) - }) + res, err := seqUser.GetReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) if err != nil { panic(err) } - t.Log("==>", res) + + t.Log(res) + } diff --git a/pkg/common/storage/cache/redis/redis_shard_manager.go b/pkg/common/storage/cache/redis/redis_shard_manager.go index 98d70dabf9..17e5fecf6f 100644 --- a/pkg/common/storage/cache/redis/redis_shard_manager.go +++ b/pkg/common/storage/cache/redis/redis_shard_manager.go @@ -2,6 +2,7 @@ package redis import ( "context" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -109,7 +110,7 @@ func (rsm *RedisShardManager) ProcessKeysBySlot( func groupKeysBySlot(ctx context.Context, redisClient redis.UniversalClient, keys []string) (map[int64][]string, error) { slots := make(map[int64][]string) clusterClient, isCluster := redisClient.(*redis.ClusterClient) - if isCluster { + if isCluster && len(keys) > 1 { pipe := clusterClient.Pipeline() cmds := make([]*redis.IntCmd, len(keys)) for i, key := range keys { @@ -195,3 +196,16 @@ func ProcessKeysBySlot( } return nil } + +func DeleteCacheBySlot(ctx context.Context, rcClient *rockscache.Client, keys []string) error { + switch len(keys) { + case 0: + return nil + case 1: + return rcClient.TagAsDeletedBatch2(ctx, keys) + default: + return ProcessKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), keys, func(ctx context.Context, slot int64, keys []string) error { + return rcClient.TagAsDeletedBatch2(ctx, keys) + }) + } +} diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index 034462fd1d..76cac2b02f 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -2,6 +2,7 @@ package redis import ( "context" + "errors" "fmt" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -39,13 +40,7 @@ func (s *seqConversationCacheRedis) getMinSeqKey(conversationID string) string { } func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { - if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { - return err - } - if err := s.rocks.TagAsDeleted2(ctx, s.getMinSeqKey(conversationID)); err != nil { - return errs.Wrap(err) - } - return nil + return s.SetMinSeqs(ctx, map[string]int64{conversationID: seq}) } func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { @@ -54,6 +49,78 @@ func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationI }) } +func (s *seqConversationCacheRedis) getSingleMaxSeq(ctx context.Context, conversationID string) (map[string]int64, error) { + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return nil, err + } + return map[string]int64{conversationID: seq}, nil +} + +func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []string, keyConversationID map[string]string, seqs map[string]int64) error { + result := make([]*redis.StringCmd, len(keys)) + pipe := s.rdb.Pipeline() + for i, key := range keys { + result[i] = pipe.HGet(ctx, key, "CURR") + } + if _, err := pipe.Exec(ctx); err != nil { + return errs.Wrap(err) + } + var notFoundKey []string + for i, r := range result { + req, err := r.Int64() + if err == nil { + seqs[keyConversationID[keys[i]]] = req + } else if errors.Is(err, redis.Nil) { + notFoundKey = append(notFoundKey, keys[i]) + } else { + return errs.Wrap(err) + } + } + if len(notFoundKey) > 0 { + conversationID := keyConversationID[notFoundKey[0]] + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return err + } + seqs[conversationID] = seq + } + return nil +} + +func (s *seqConversationCacheRedis) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { + switch len(conversationIDs) { + case 0: + return map[string]int64{}, nil + case 1: + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + keys := make([]string, 0, len(conversationIDs)) + keyConversationID := make(map[string]string, len(conversationIDs)) + for _, conversationID := range conversationIDs { + key := s.getSeqMallocKey(conversationID) + if _, ok := keyConversationID[key]; ok { + continue + } + keys = append(keys, key) + keyConversationID[key] = conversationID + } + if len(keys) == 1 { + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return nil, err + } + seqs := make(map[string]int64, len(conversationIDs)) + for _, keys := range slotKeys { + if err := s.batchGetMaxSeq(ctx, keys, keyConversationID, seqs); err != nil { + return nil, err + } + } + return seqs, nil +} + func (s *seqConversationCacheRedis) getSeqMallocKey(conversationID string) string { return cachekey.GetMallocSeqKey(conversationID) } @@ -253,3 +320,14 @@ func (s *seqConversationCacheRedis) Malloc(ctx context.Context, conversationID s func (s *seqConversationCacheRedis) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { return s.Malloc(ctx, conversationID, 0) } + +func (s *seqConversationCacheRedis) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + keys = append(keys, s.getMinSeqKey(conversationID)) + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index 3c533cdd83..2ad43eebdb 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -64,10 +64,7 @@ func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string } func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMinSeqKey(conversationID, userID)) + return s.SetMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) } func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { @@ -87,3 +84,102 @@ func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID strin } return nil } + +func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + keys = append(keys, s.getSeqUserMinSeqKey(conversationID, userID)) + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} + +func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + keySeq := make(map[string]int64) + for conversationID, seq := range seqs { + key := s.getSeqUserReadSeqKey(conversationID, userID) + keys = append(keys, key) + keySeq[key] = seq + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return err + } + for _, keys := range slotKeys { + pipe := s.rdb.Pipeline() + for _, key := range keys { + pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10)) + pipe.Expire(ctx, key, s.readExpireTime) + } + if _, err := pipe.Exec(ctx); err != nil { + return err + } + } + return nil +} + +func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + if len(seqs) == 0 { + return nil + } + if err := s.setRedisReadSeqs(ctx, userID, seqs); err != nil { + return err + } + for conversationID, seq := range seqs { + if seq%s.readSeqWriteRatio == 0 { + if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + } + return nil +} + +func (s *seqUserCacheRedis) GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { + res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { + return s.getSeqUserReadSeqKey(conversationID, userID) + }, func(v *readSeqModel) string { + return v.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { + seqs, err := s.mgo.GetReadSeqs(ctx, userID, conversationIDs) + if err != nil { + return nil, err + } + res := make([]*readSeqModel, 0, len(seqs)) + for conversationID, seq := range seqs { + res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq}) + } + return res, nil + }) + if err != nil { + return nil, err + } + data := make(map[string]int64) + for _, v := range res { + data[v.ConversationID] = v.Seq + } + return data, nil +} + +var _ BatchCacheCallback[string] = (*readSeqModel)(nil) + +type readSeqModel struct { + ConversationID string + Seq int64 +} + +func (r *readSeqModel) BatchCache(conversationID string) { + r.ConversationID = conversationID +} + +func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) { + r.Seq, err = strconv.ParseInt(string(bytes), 10, 64) + return +} + +func (r *readSeqModel) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatInt(r.Seq, 10)), nil +} diff --git a/pkg/common/storage/cache/seq_conversation.go b/pkg/common/storage/cache/seq_conversation.go index 5d38537a9e..2c893a5e86 100644 --- a/pkg/common/storage/cache/seq_conversation.go +++ b/pkg/common/storage/cache/seq_conversation.go @@ -7,4 +7,6 @@ type SeqConversationCache interface { GetMaxSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeq(ctx context.Context, conversationID string, seq int64) error GetMinSeq(ctx context.Context, conversationID string) (int64, error) + GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + SetMinSeqs(ctx context.Context, seqs map[string]int64) error } diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go index 9e68399a73..4d0bb4ffab 100644 --- a/pkg/common/storage/cache/seq_user.go +++ b/pkg/common/storage/cache/seq_user.go @@ -9,4 +9,7 @@ type SeqUser interface { SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error + SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error + GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 1e7b5c2290..32202ac9e9 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -72,15 +72,8 @@ type CommonMsgDatabase interface { //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error SetMinSeqs(ctx context.Context, seqs map[string]int64) error - //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - //GetMinSeq(ctx context.Context, conversationID string) (int64, error) - //GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - //GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - //SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - //SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) @@ -784,44 +777,8 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st return nil } -func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationID string, seqs []int64) error { - return nil -} - -func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) { - for _, conversationID := range conversationIDs { - maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) - if err != nil { - if err == redis.Nil { - log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID) - } else { - log.ZError(ctx, "get max seq failed", err, "conversationID", conversationID) - } - continue - } - if err := db.seqConversation.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { - log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1) - } - } -} - -//func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { -// return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) -//} - func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - result := make(map[string]int64) - for _, conversationID := range conversationIDs { - if result[conversationID] != 0 { - continue - } - seq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) - if err != nil { - return nil, err - } - result[conversationID] = seq - } - return result, nil + return db.seqConversation.GetMaxSeqs(ctx, conversationIDs) } func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { @@ -833,30 +790,15 @@ func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID strin } func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - for conversationID, seq := range seqs { - if err := db.seqConversation.SetMinSeq(ctx, conversationID, seq); err != nil { - return err - } - } - return nil + return db.seqConversation.SetMinSeqs(ctx, seqs) } func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - for conversationID, seq := range seqs { - if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - } - return nil + return db.seqUser.SetMinSeqs(ctx, userID, seqs) } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - for conversationID, seq := range hasReadSeqs { - if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - } - return nil + return db.seqUser.SetReadSeqs(ctx, userID, hasReadSeqs) } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { @@ -864,18 +806,7 @@ func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, c } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - cSeq := make(map[string]int64) - for _, conversationID := range conversationIDs { - if _, ok := cSeq[conversationID]; ok { - continue - } - seq, err := db.seqUser.GetReadSeq(ctx, conversationID, userID) - if err != nil { - return nil, err - } - cSeq[conversationID] = seq - } - return cSeq, nil + return db.seqUser.GetReadSeqs(ctx, userID, conversationIDs) } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 5e0eb2022d..e0cbb08d9c 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -87,6 +88,23 @@ func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, us return s.getSeq(ctx, conversationID, userID, "read_seq") } +func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { + if len(conversationID) == 0 { + return map[string]int64{}, nil + } + filter := bson.M{"user_id": userID, "conversation_id": bson.M{"$in": conversationID}} + opt := options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1, "read_seq": 1}) + seqs, err := mongoutil.Find[*model.SeqUser](ctx, s.coll, filter, opt) + if err != nil { + return nil, err + } + res := make(map[string]int64) + for _, seq := range seqs { + res[seq.ConversationID] = seq.ReadSeq + } + return res, nil +} + func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go index d32d614ddd..edd3910d02 100644 --- a/pkg/common/storage/database/seq_user.go +++ b/pkg/common/storage/database/seq_user.go @@ -9,4 +9,5 @@ type SeqUser interface { SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) } From 3d7090e2f698cb08515871f4ce572d4fa62d7111 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 15 Jul 2024 11:05:34 +0800 Subject: [PATCH 118/147] fix: GetMsgDocModelByIndex bug --- pkg/common/storage/database/mgo/msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 04aa59cef9..ebabadf52c 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -205,7 +205,7 @@ func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID strin if sort != 1 && sort != -1 { return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1") } - opt := options.Find().SetLimit(1).SetSkip(index).SetSort(bson.M{"doc_id": sort}).SetLimit(1) + opt := options.Find().SetSkip(index).SetSort(bson.M{"_id": sort}).SetLimit(1) filter := bson.M{"doc_id": primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)}} msgs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, filter, opt) if err != nil { From e4fb5baf44c2865f718d3c47648cc5a97e48e7f5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 15 Jul 2024 11:20:34 +0800 Subject: [PATCH 119/147] update go.mod --- go.mod | 2 +- go.sum | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 63fe0d47ec..207c2cf5e7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.30 - github.com/openimsdk/tools v0.0.49-alpha.45 + github.com/openimsdk/tools v0.0.49-alpha.49 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index dd6069abe0..ef8f68a912 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,42 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= +github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= +github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= +github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 h1:1w11lfXOa8HoHoSlNtt4mqv/N3HmDOa+OnUH3Y9DHm8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -49,6 +85,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -95,12 +132,18 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -214,10 +257,16 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= @@ -264,12 +313,13 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= -github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/tools v0.0.49-alpha.49 h1:bkVxlEDM9rJxo5a98mk2ETR0xbEv+5mjeVVVFQfARAQ= +github.com/openimsdk/tools v0.0.49-alpha.49/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -286,12 +336,18 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -324,6 +380,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -396,7 +453,9 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= @@ -423,6 +482,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= @@ -445,20 +505,26 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -515,8 +581,10 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -527,6 +595,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= From 68c41ba432c99cba334dd545bd8da378c5e92b95 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 15 Jul 2024 15:04:03 +0800 Subject: [PATCH 120/147] update go.mod --- go.mod | 2 +- go.sum | 4 ++-- internal/api/router.go | 2 -- internal/rpc/third/s3.go | 4 ++-- internal/rpc/third/third.go | 16 +++++++++++++--- pkg/common/storage/controller/s3.go | 5 ----- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 13eab9c3f8..2e47d1b5e5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.30 - github.com/openimsdk/tools v0.0.49-alpha.49 + github.com/openimsdk/tools v0.0.49-alpha.50 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index ef8f68a912..649150dec7 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.49 h1:bkVxlEDM9rJxo5a98mk2ETR0xbEv+5mjeVVVFQfARAQ= -github.com/openimsdk/tools v0.0.49-alpha.49/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/tools v0.0.49-alpha.50 h1:7CaYLVtsBU5kyiTetUOuOkO5FFFmMvSzBEfh2tfCn90= +github.com/openimsdk/tools v0.0.49-alpha.50/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/router.go b/internal/api/router.go index 5a5b24ddaf..0667c3e751 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -16,8 +16,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "net/http" "strings" ) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 54cbc30c1a..f96eb73905 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -310,8 +310,8 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { return nil, errs.Wrap(err) } - if int(count) < 1 { - thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + if int(count) < 1 && t.minio != nil { + thumbnailKey, err := t.getMinioImageThumbnailKey(ctx, key) if err != nil { return nil, errs.Wrap(err) } diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 3ae5600330..0eeaaa3140 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -31,9 +31,9 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "github.com/openimsdk/tools/s3/kodo" "google.golang.org/grpc" ) @@ -43,6 +43,7 @@ type thirdServer struct { userRpcClient rpcclient.UserRpcClient defaultExpire time.Duration config *Config + minio *minio.Minio } type Config struct { @@ -75,10 +76,14 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable - var o s3.Interface + var ( + o s3.Interface + minioCli *minio.Minio + ) switch enable { case "minio": - o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + o = minioCli case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -98,10 +103,15 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, + minio: minioCli, }) return nil } +func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { + return t.minio.GetImageThumbnailKey(ctx, name) +} + func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 5926595360..9b56661a59 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -45,7 +45,6 @@ type S3Database interface { DeleteSpecifiedData(ctx context.Context, engine string, name string) error FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) DelS3Key(ctx context.Context, engine string, keys ...string) error - GetImageThumbnailKey(ctx context.Context, name string) (string, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -140,7 +139,3 @@ func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration ti func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { return s.s3cache.DelS3Key(ctx, engine, keys...) } - -func (s *s3Database) GetImageThumbnailKey(ctx context.Context, name string) (string, error) { - return s.s3.GetImageThumbnailKey(ctx, name) -} From 6a6c712f52aaa4fa4b4808a02aa5884e9a391c7e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 15 Jul 2024 16:12:44 +0800 Subject: [PATCH 121/147] merge --- go.sum | 4 ++-- internal/api/router.go | 3 --- internal/rpc/third/s3.go | 5 ----- internal/rpc/user/user.go | 5 ----- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/go.sum b/go.sum index ef8f68a912..649150dec7 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.49 h1:bkVxlEDM9rJxo5a98mk2ETR0xbEv+5mjeVVVFQfARAQ= -github.com/openimsdk/tools v0.0.49-alpha.49/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/tools v0.0.49-alpha.50 h1:7CaYLVtsBU5kyiTetUOuOkO5FFFmMvSzBEfh2tfCn90= +github.com/openimsdk/tools v0.0.49-alpha.50/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/router.go b/internal/api/router.go index 8ffd24edf3..0667c3e751 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -8,9 +8,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 8864bfa551..f96eb73905 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -340,8 +340,3 @@ type FormDataMate struct { Group string `json:"group"` Key string `json:"key"` } - -func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - //TODO implement me - panic("implement me") -} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 75d01d2615..0b96077ec3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -669,8 +669,3 @@ func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (* } return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } - -func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { - //TODO implement me - panic("implement me") -} From 7f4d6def53163bac9bc542364fd7c759f73dd35a Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Mon, 15 Jul 2024 18:04:54 +0800 Subject: [PATCH 122/147] feat: prometheus --- .env | 3 + .gitignore | 4 -- config/alertmanager.yml | 25 +++++++ config/email.tmpl | 16 +++++ config/instance-down-rules.yml | 22 ++++++ config/prometheus.yml | 83 ++++++++++++++++++++++ docker-compose.yml | 45 ++++++++++++ internal/api/init.go | 4 +- internal/msgtransfer/init.go | 15 +--- pkg/common/prommetrics/api.go | 20 +++--- pkg/common/prommetrics/prommetrics.go | 47 ++++-------- pkg/common/prommetrics/prommetrics_test.go | 73 +++++++++---------- pkg/common/prommetrics/rpc.go | 39 ++++++++-- pkg/common/prommetrics/transfer.go | 14 ++++ pkg/common/startrpc/start.go | 23 +++--- 15 files changed, 315 insertions(+), 118 deletions(-) create mode 100644 config/alertmanager.yml create mode 100644 config/email.tmpl create mode 100644 config/instance-down-rules.yml create mode 100644 config/prometheus.yml diff --git a/.env b/.env index 1e7b1e11a7..3199b37149 100644 --- a/.env +++ b/.env @@ -5,6 +5,9 @@ ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 +PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 +ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 +GRAFANA_IMAGE=grafana/grafana:11.0.1 OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.5.1 OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.7 diff --git a/.gitignore b/.gitignore index fb8d428d24..77cf855b70 100644 --- a/.gitignore +++ b/.gitignore @@ -34,11 +34,7 @@ deployments/charts/generated-configs/ ### OpenIM Config ### .env config/config.yaml -config/alertmanager.yml -config/prometheus.yml -config/email.tmpl config/notification.yaml -config/instance-down-rules.yml ### OpenIM deploy ### deployments/openim-server/charts diff --git a/config/alertmanager.yml b/config/alertmanager.yml new file mode 100644 index 0000000000..bdb076a63b --- /dev/null +++ b/config/alertmanager.yml @@ -0,0 +1,25 @@ +global: + resolve_timeout: 5m + smtp_from: alert@openim.io + smtp_smarthost: smtp.163.com:465 + smtp_auth_username: alert@openim.io + smtp_auth_password: YOURAUTHPASSWORD + smtp_require_tls: false + smtp_hello: xxx监控告警 + +templates: + - /etc/alertmanager/email.tmpl + +route: + group_by: ['alertname'] + group_wait: 5s + group_interval: 5s + repeat_interval: 5m + receiver: email +receivers: + - name: email + email_configs: + - to: 'alert@example.com' + html: '{{ template "email.to.html" . }}' + headers: { Subject: "[OPENIM-SERVER]Alarm" } + send_resolved: true diff --git a/config/email.tmpl b/config/email.tmpl new file mode 100644 index 0000000000..0385601d00 --- /dev/null +++ b/config/email.tmpl @@ -0,0 +1,16 @@ +{{ define "email.to.html" }} +{{ range .Alerts }} + +
+

OpenIM Alert

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ +{{ end }} +{{ end }} diff --git a/config/instance-down-rules.yml b/config/instance-down-rules.yml new file mode 100644 index 0000000000..5541d2c542 --- /dev/null +++ b/config/instance-down-rules.yml @@ -0,0 +1,22 @@ +groups: + - name: instance_down + rules: + - alert: InstanceDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes." + + - name: database_insert_failure_alerts + rules: + - alert: DatabaseInsertFailed + expr: (increase(msg_insert_redis_failed_total[5m]) > 0) or (increase(msg_insert_mongo_failed_total[5m]) > 0) + for: 1m + labels: + severity: critical + annotations: + summary: "Increase in MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter detected" + description: "Either MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter has increased in the last 5 minutes, indicating failures in message insert operations to Redis or MongoDB,maybe the redis or mongodb is crash." diff --git a/config/prometheus.yml b/config/prometheus.yml new file mode 100644 index 0000000000..99f2e4df34 --- /dev/null +++ b/config/prometheus.yml @@ -0,0 +1,83 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: ['192.168.2.22:19093'] + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "instance-down-rules.yml" +# - "first_rules.yml" +# - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label "job='job_name'"" to any timeseries scraped from this config. + # Monitored information captured by prometheus + + # prometheus fetches application services + - job_name: 'node_exporter' + static_configs: + - targets: [ '192.168.2.22:20114' ] + - job_name: 'openimserver-openim-api' + static_configs: + - targets: [ '192.168.2.22:20113' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-msggateway' + static_configs: + - targets: [ '192.168.2.22:20112' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-msgtransfer' + static_configs: + - targets: [ 192.168.2.22:20111, 192.168.2.22:20110, 192.168.2.22:20109, 192.168.2.22:20108 ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-push' + static_configs: + - targets: [ '192.168.2.22:20107' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-auth' + static_configs: + - targets: [ '192.168.2.22:20106' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-conversation' + static_configs: + - targets: [ '192.168.2.22:20105' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-friend' + static_configs: + - targets: [ '192.168.2.22:20104' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-group' + static_configs: + - targets: [ '192.168.2.22:20103' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-msg' + static_configs: + - targets: [ '192.168.2.22:20102' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-third' + static_configs: + - targets: [ '192.168.2.22:20101' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-user' + static_configs: + - targets: [ '192.168.2.22:20100' ] + labels: + namespace: 'default' \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d72c1a2fa4..8cc1f24b2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,5 +140,50 @@ services: networks: - openim + prometheus: + image: ${PROMETHEUS_IMAGE} + container_name: prometheus + restart: always + volumes: + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml + - ${DATA_DIR}/components/prometheus/data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + ports: + - "19091:9090" + networks: + - openim + + alertmanager: + image: ${ALERTMANAGER_IMAGE} + container_name: alertmanager + restart: always + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + - ./config/email.tmpl:/etc/alertmanager/email.tmpl + ports: + - "19093:9093" + networks: + - openim + + grafana: + image: ${GRAFANA_IMAGE} + container_name: grafana + user: root + restart: always + environment: + - GF_SECURITY_ALLOW_EMBEDDING=true + - GF_SESSION_COOKIE_SAMESITE=none + - GF_SESSION_COOKIE_SECURE=true + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "13000:3000" + volumes: + - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana + networks: + - openim diff --git a/internal/api/init.go b/internal/api/init.go index 679d901f4b..e83dfc2ea5 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -71,9 +71,7 @@ func Start(ctx context.Context, index int, config *Config) error { netDone <- struct{}{} return } - srv := http.NewServeMux() - srv.Handle(prommetrics.ApiPath, prommetrics.ApiHandler()) - if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { + if err := prommetrics.ApiInit(prometheusPort); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index c17abbc305..b4b2245eb0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -17,6 +17,7 @@ package msgtransfer import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" @@ -29,16 +30,12 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -139,14 +136,8 @@ func (m *MsgTransfer) Start(index int, config *Config) error { netDone <- struct{}{} return } - proreg := prometheus.NewRegistry() - proreg.MustRegister( - collectors.NewGoCollector(), - ) - proreg.MustRegister(prommetrics.GetGrpcCusMetrics("Transfer", &config.Share)...) - http.Handle("/metrics", promhttp.HandlerFor(proreg, promhttp.HandlerOpts{Registry: proreg})) - err = http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), nil) - if err != nil && err != http.ErrServerClosed { + + if err := prommetrics.TransferInit(prometheusPort); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 8061989770..95b5c06b68 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -3,15 +3,11 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" "strconv" ) -const ApiPath = "/metrics" - var ( - apiRegistry = prometheus.NewRegistry() - apiCounter = prometheus.NewCounterVec( + apiCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "api_count", Help: "Total number of API calls", @@ -27,8 +23,14 @@ var ( ) ) -func init() { - apiRegistry.MustRegister(apiCounter, httpCounter) +func ApiInit(prometheusPort int) error { + apiRegistry := prometheus.NewRegistry() + cs := append( + baseCollector, + apiCounter, + httpCounter, + ) + return Init(apiRegistry, prometheusPort, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) } func APICall(path string, method string, apiCode int) { @@ -44,7 +46,3 @@ func HttpCall(path string, method string, status int) { // apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), // ) //} - -func ApiHandler() http.Handler { - return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) -} diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 3955d8ea5f..02e408d63b 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,43 +15,24 @@ package prommetrics import ( - gp "github.com/grpc-ecosystem/go-grpc-prometheus" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "net/http" ) -func NewGrpcPromObj(cusMetrics []prometheus.Collector) (*prometheus.Registry, *gp.ServerMetrics, error) { - reg := prometheus.NewRegistry() - grpcMetrics := gp.NewServerMetrics() - grpcMetrics.EnableHandlingTimeHistogram() - cusMetrics = append(cusMetrics, grpcMetrics, collectors.NewGoCollector()) - reg.MustRegister(cusMetrics...) - return reg, grpcMetrics, nil -} +const commonPath = "/metrics" -func GetGrpcCusMetrics(registerName string, share *config2.Share) []prometheus.Collector { - switch registerName { - case share.RpcRegisterName.MessageGateway: - return []prometheus.Collector{OnlineUserGauge} - case share.RpcRegisterName.Msg: - return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} - case "Transfer": - return []prometheus.Collector{MsgInsertRedisSuccessCounter, MsgInsertRedisFailedCounter, MsgInsertMongoSuccessCounter, MsgInsertMongoFailedCounter, SeqSetFailedCounter} - case share.RpcRegisterName.Push: - return []prometheus.Collector{MsgOfflinePushFailedCounter} - case share.RpcRegisterName.Auth: - return []prometheus.Collector{UserLoginCounter} - default: - return nil +var ( + baseCollector = []prometheus.Collector{ + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), } -} +) -//func GetGinCusMetrics(name string) []*ginprometheus.Metric { -// switch name { -// case "Api": -// return []*ginprometheus.Metric{ApiCustomCnt} -// default: -// return []*ginprometheus.Metric{ApiCustomCnt} -// } -//} +func Init(registry *prometheus.Registry, prometheusPort int, path string, handler http.Handler, cs ...prometheus.Collector) error { + registry.MustRegister(cs...) + srv := http.NewServeMux() + srv.Handle(path, handler) + return http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv) +} diff --git a/pkg/common/prommetrics/prommetrics_test.go b/pkg/common/prommetrics/prommetrics_test.go index 65b05652f7..14b1aaff32 100644 --- a/pkg/common/prommetrics/prommetrics_test.go +++ b/pkg/common/prommetrics/prommetrics_test.go @@ -14,46 +14,39 @@ package prommetrics -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" -) - -func TestNewGrpcPromObj(t *testing.T) { - // Create a custom metric to pass into the NewGrpcPromObj function. - customMetric := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "test_metric", - Help: "This is a test metric.", - }) - cusMetrics := []prometheus.Collector{customMetric} - - // Call NewGrpcPromObj with the custom metrics. - reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) - - // Assert no error was returned. - assert.NoError(t, err) - - // Assert the registry was correctly initialized. - assert.NotNil(t, reg) - - // Assert the grpcMetrics was correctly initialized. - assert.NotNil(t, grpcMetrics) - - // Assert that the custom metric is registered. - mfs, err := reg.Gather() - assert.NoError(t, err) - assert.NotEmpty(t, mfs) // Ensure some metrics are present. - found := false - for _, mf := range mfs { - if *mf.Name == "test_metric" { - found = true - break - } - } - assert.True(t, found, "Custom metric not found in registry") -} +//func TestNewGrpcPromObj(t *testing.T) { +// // Create a custom metric to pass into the NewGrpcPromObj function. +// customMetric := prometheus.NewCounter(prometheus.CounterOpts{ +// Name: "test_metric", +// Help: "This is a test metric.", +// }) +// cusMetrics := []prometheus.Collector{customMetric} +// +// // Call NewGrpcPromObj with the custom metrics. +// reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) +// +// // Assert no error was returned. +// assert.NoError(t, err) +// +// // Assert the registry was correctly initialized. +// assert.NotNil(t, reg) +// +// // Assert the grpcMetrics was correctly initialized. +// assert.NotNil(t, grpcMetrics) +// +// // Assert that the custom metric is registered. +// mfs, err := reg.Gather() +// assert.NoError(t, err) +// assert.NotEmpty(t, mfs) // Ensure some metrics are present. +// found := false +// for _, mf := range mfs { +// if *mf.Name == "test_metric" { +// found = true +// break +// } +// } +// assert.True(t, found, "Custom metric not found in registry") +//} //func TestGetGrpcCusMetrics(t *testing.T) { // conf := config2.NewGlobalConfig() diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 834f76a9fb..1da2c1510d 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -1,16 +1,17 @@ package prommetrics import ( + gp "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" "strconv" ) -const RpcPath = "/metrics" +const rpcPath = commonPath var ( - rpcRegistry = prometheus.NewRegistry() + grpcMetrics *gp.ServerMetrics rpcCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "rpc_count", @@ -20,14 +21,38 @@ var ( ) ) -func init() { - rpcRegistry.MustRegister(rpcCounter) +func RpcInit(cs []prometheus.Collector, prometheusPort int) error { + reg := prometheus.NewRegistry() + cs = append(append( + baseCollector, + rpcCounter, + ), cs...) + return Init(reg, prometheusPort, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) } func RPCCall(name string, path string, code int) { rpcCounter.With(prometheus.Labels{"name": name, "path": path, "code": strconv.Itoa(code)}).Inc() } -func RPCHandler() http.Handler { - return promhttp.HandlerFor(rpcRegistry, promhttp.HandlerOpts{}) +func GetGrpcServerMetrics() *gp.ServerMetrics { + if grpcMetrics == nil { + grpcMetrics = gp.NewServerMetrics() + grpcMetrics.EnableHandlingTimeHistogram() + } + return grpcMetrics +} + +func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Collector { + switch registerName { + case share.RpcRegisterName.MessageGateway: + return []prometheus.Collector{OnlineUserGauge} + case share.RpcRegisterName.Msg: + return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} + case share.RpcRegisterName.Push: + return []prometheus.Collector{MsgOfflinePushFailedCounter} + case share.RpcRegisterName.Auth: + return []prometheus.Collector{UserLoginCounter} + default: + return nil + } } diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index 197b6f7fc8..f0abb82856 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -16,6 +16,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -40,3 +41,16 @@ var ( Help: "The number of failed set seq", }) ) + +func TransferInit(prometheusPort int) error { + reg := prometheus.NewRegistry() + cs := append( + baseCollector, + MsgInsertRedisSuccessCounter, + MsgInsertRedisFailedCounter, + MsgInsertMongoSuccessCounter, + MsgInsertMongoFailedCounter, + SeqSetFailedCounter, + ) + return Init(reg, prometheusPort, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index a88797793c..4091a5f6e6 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -17,7 +17,7 @@ package startrpc import ( "context" "fmt" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc/status" "net" @@ -42,8 +42,8 @@ import ( ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, - registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, + registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) @@ -82,7 +82,11 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) - options = append(options, mw.GrpcServer(), prommetricsInterceptor(rpcRegisterName)) + options = append( + options, mw.GrpcServer(), + prommetricsUnaryInterceptor(rpcRegisterName), + prommetricsStreamInterceptor(rpcRegisterName), + ) } else { options = append(options, mw.GrpcServer()) } @@ -121,9 +125,8 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC netDone <- struct{}{} return } - srv := http.NewServeMux() - srv.Handle(prommetrics.RpcPath, prommetrics.RPCHandler()) - if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } @@ -181,7 +184,7 @@ func gracefulStopWithCtx(ctx context.Context, f func()) error { } } -func prommetricsInterceptor(rpcRegisterName string) grpc.ServerOption { +func prommetricsUnaryInterceptor(rpcRegisterName string) grpc.ServerOption { getCode := func(err error) int { if err == nil { return 0 @@ -198,3 +201,7 @@ func prommetricsInterceptor(rpcRegisterName string) grpc.ServerOption { return resp, err }) } + +func prommetricsStreamInterceptor(rpcRegisterName string) grpc.ServerOption { + return grpc.ChainStreamInterceptor() +} From 4738d1729cf2b1ff8020a819082cee9dabc2ca70 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Mon, 15 Jul 2024 18:26:58 +0800 Subject: [PATCH 123/147] feat: prometheus --- config/alertmanager.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/alertmanager.yml b/config/alertmanager.yml index bdb076a63b..a029448518 100644 --- a/config/alertmanager.yml +++ b/config/alertmanager.yml @@ -5,7 +5,7 @@ global: smtp_auth_username: alert@openim.io smtp_auth_password: YOURAUTHPASSWORD smtp_require_tls: false - smtp_hello: xxx监控告警 + smtp_hello: xxx templates: - /etc/alertmanager/email.tmpl From 059902020ddf6538bc671ca826c7525bdaee2bd1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 17 Jul 2024 10:27:59 +0800 Subject: [PATCH 124/147] group member sort --- pkg/common/storage/database/mgo/group_member.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index f89822d3c4..8de21e83a5 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -59,7 +59,7 @@ type GroupMemberMgo struct { } func (g *GroupMemberMgo) memberSort() any { - return bson.D{{"role_level", -1}, {"create_time", -1}} + return bson.D{{"role_level", -1}, {"create_time", 1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { From b6ab4a5f8465b1ceb725d86fdb6b702314a5bb96 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 17 Jul 2024 16:20:01 +0800 Subject: [PATCH 125/147] sub --- internal/msggateway/client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 446553dc08..d5368edf42 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -94,6 +94,11 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closedErr = nil c.token = ctx.GetToken() c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) + if c.subUserIDs == nil { + c.subUserIDs = make(map[string]struct{}) + } else { + clear(c.subUserIDs) + } } func (c *Client) pingHandler(_ string) error { From bfca6ced2a7fe432ed7b0e856cf43dc027a1b651 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 17 Jul 2024 18:03:10 +0800 Subject: [PATCH 126/147] sub --- internal/msggateway/client.go | 24 +++++++---- internal/msggateway/subscription.go | 65 +++++++++++------------------ internal/msggateway/ws_server.go | 4 +- 3 files changed, 41 insertions(+), 52 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index d5368edf42..289d4e94d9 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -75,8 +75,8 @@ type Client struct { token string hbCtx context.Context hbCancel context.CancelFunc - subLock sync.Mutex - subUserIDs map[string]struct{} + subLock *sync.Mutex + subUserIDs map[string]struct{} // client conn subscription list } // ResetClient updates the client's state with new connection and context information. @@ -94,11 +94,11 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closedErr = nil c.token = ctx.GetToken() c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) - if c.subUserIDs == nil { - c.subUserIDs = make(map[string]struct{}) - } else { + c.subLock = new(sync.Mutex) + if c.subUserIDs != nil { clear(c.subUserIDs) } + c.subUserIDs = make(map[string]struct{}) } func (c *Client) pingHandler(_ string) error { @@ -249,13 +249,11 @@ func (c *Client) setAppBackgroundStatus(ctx context.Context, req *Req) ([]byte, } func (c *Client) close() { + c.w.Lock() + defer c.w.Unlock() if c.closed.Load() { return } - - c.w.Lock() - defer c.w.Unlock() - c.closed.Store(true) c.conn.Close() c.hbCancel() // Close server-initiated heartbeat. @@ -316,6 +314,14 @@ func (c *Client) KickOnlineMessage() error { return err } +func (c *Client) PushUserOnlineStatus(data []byte) error { + resp := Resp{ + ReqIdentifier: WsSubUserOnlineStatus, + Data: data, + } + return c.writeBinaryMsg(resp) +} + func (c *Client) writeBinaryMsg(resp Resp) error { if c.closed.Load() { return nil diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 9460f5dbfb..9bb41e0dfd 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -2,15 +2,11 @@ package msggateway import ( "context" - "encoding/json" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/idutil" "google.golang.org/protobuf/proto" "sync" - "time" ) func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { @@ -45,33 +41,19 @@ func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, dat return proto.Marshal(&resp) } -type subClient struct { - clients map[string]*Client -} - func newSubscription() *Subscription { return &Subscription{ userIDs: make(map[string]*subClient), } } -type Subscription struct { - lock sync.RWMutex - userIDs map[string]*subClient +type subClient struct { + clients map[string]*Client } -func (s *Subscription) GetClient(userID string) []*Client { - s.lock.RLock() - defer s.lock.RUnlock() - cs, ok := s.userIDs[userID] - if !ok { - return nil - } - clients := make([]*Client, 0, len(cs.clients)) - for _, client := range cs.clients { - clients = append(clients, client) - } - return clients +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient // subscribe to the user's client connection } func (s *Subscription) DelClient(client *Client) { @@ -99,6 +81,20 @@ func (s *Subscription) DelClient(client *Client) { } } +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { if len(addUserIDs)+len(delUserIDs) == 0 { return @@ -121,6 +117,7 @@ func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { continue } client.subUserIDs[userID] = struct{}{} + add[userID] = struct{}{} } client.subLock.Unlock() if len(del)+len(add) == 0 { @@ -154,28 +151,16 @@ func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, p if len(clients) == 0 { return } - msgContent, err := json.Marshal(platformIDs) + onlineStatus, err := proto.Marshal(&sdkws.SubUserOnlineStatusTips{ + Subscribers: []*sdkws.SubUserOnlineStatusElem{{UserID: userID, OnlinePlatformIDs: platformIDs}}, + }) if err != nil { log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) return } - now := time.Now().UnixMilli() - msgID := idutil.GetMsgIDByMD5(userID) - msg := &sdkws.MsgData{ - SendID: userID, - ClientMsgID: msgID, - ServerMsgID: msgID, - SenderPlatformID: constant.AdminPlatformID, - SessionType: constant.NotificationChatType, - ContentType: constant.UserSubscribeOnlineStatusNotification, - Content: msgContent, - SendTime: now, - CreateTime: now, - } for _, client := range clients { - msg.RecvID = client.UserID - if err := client.PushMessage(ctx, msg); err != nil { - log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) + if err := client.PushUserOnlineStatus(onlineStatus); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "changePlatformID", platformIDs) } } } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index e903084a9d..537b8c5f0a 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -358,9 +358,7 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - client.subLock.Lock() - clear(client.subUserIDs) - client.subLock.Unlock() + ws.subscription.DelClient(client) //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", From 2ce8d221898e221d537069e4f06baa90313b622a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 17 Jul 2024 20:31:21 +0800 Subject: [PATCH 127/147] fix: seq conversion bug --- tools/seq/internal/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 5200ad5fef..7f021a90e8 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -116,7 +116,7 @@ func Main(conf string, del time.Duration) error { { Prefix: MaxSeq, GetSeq: cSeq.GetMaxSeq, - SetSeq: cSeq.SetMinSeq, + SetSeq: cSeq.SetMaxSeq, }, { Prefix: MinSeq, From d521f0b0311a4c15230d29513a9ccb220841e1f1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 17 Jul 2024 21:43:10 +0800 Subject: [PATCH 128/147] fix: redis pipe exec --- pkg/common/storage/cache/redis/seq_conversation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index 76cac2b02f..fb8a547dfa 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -63,7 +63,7 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s for i, key := range keys { result[i] = pipe.HGet(ctx, key, "CURR") } - if _, err := pipe.Exec(ctx); err != nil { + if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { return errs.Wrap(err) } var notFoundKey []string From ae06f16c5a32183d3b2034b4122bccb9d7d1306d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 16:58:46 +0800 Subject: [PATCH 129/147] sort version --- go.mod | 2 +- go.sum | 4 +- internal/rpc/friend/notification.go | 29 ++++++++++++++ internal/rpc/friend/sync.go | 39 +++++++++++++------ internal/rpc/group/notification.go | 17 +++++++- internal/rpc/group/sync.go | 35 ++++++++++------- .../storage/database/mgo/group_member.go | 13 +++++-- .../storage/database/mgo/version_log.go | 20 +++++++++- .../storage/database/mgo/version_test.go | 14 +++---- pkg/common/storage/model/version_log.go | 5 +++ 10 files changed, 137 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index a799f2c086..0637492eb5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.30 + github.com/openimsdk/protocol v0.0.69-alpha.38 github.com/openimsdk/tools v0.0.49-alpha.51 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index d9f9488741..92a783c069 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= -github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.38 h1:kVZCHIXg/el8YJFoIBWhZu1sbbTUqmzgF4l0W3sUH24= +github.com/openimsdk/protocol v0.0.69-alpha.38/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.51 h1:JTPEetVSNOczw1n+XjiPozaH2SBPQAc+9VlPE41wEeY= github.com/openimsdk/tools v0.0.49-alpha.51/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index ddee025bb6..5fb34577fc 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -16,6 +16,8 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -191,10 +193,37 @@ func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context f.Notification(ctx, req.OwnerUserID, req.FriendUserID, constant.FriendDeletedNotification, &tips) } +func (f *FriendNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + return + } + } +} + +func (f *FriendNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == relationtb.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (f *FriendNotificationSender) FriendRemarkSetNotification(ctx context.Context, fromUserID, toUserID string) { tips := sdkws.FriendInfoChangedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = fromUserID tips.FromToUserID.ToUserID = toUserID + f.setSortVersion(ctx, &tips.FriendVersion, &tips.FriendVersionID, database.FriendVersionName, toUserID, &tips.FriendSortVersion) f.Notification(ctx, fromUserID, toUserID, constant.FriendRemarkSetNotification, &tips) } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 684894609c..eee9f2afdc 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" + "slices" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -52,12 +53,27 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } + var sortVersion uint64 opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ - Ctx: ctx, - VersionKey: req.UserID, - VersionID: req.VersionID, - VersionNumber: req.Version, - Version: s.db.FindFriendIncrVersion, + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindFriendIncrVersion(ctx, ownerUserID, version, limit) + if err != nil { + return nil, err + } + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == model.VersionSortChangeID { + vl.LogLen-- + sortVersion = uint64(elem.Version) + return true + } + return false + }) + return vl, nil + }, CacheMaxVersion: s.db.FindMaxFriendVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) @@ -65,12 +81,13 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { return &relation.GetIncrementalFriendsResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - Delete: deleteIds, - Insert: insertList, - Update: updateList, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, } }, } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index a8824962dc..a7398795fa 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -306,6 +306,21 @@ func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint6 } } +func (g *GroupNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == model.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -707,7 +722,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.setSortVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID, &tips.GroupSortVersion) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 75d060c0e5..12598510be 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -10,7 +10,6 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" - "slices" ) func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { @@ -63,7 +62,10 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } - var hasGroupUpdate bool + var ( + hasGroupUpdate bool + sortVersion uint64 + ) opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, VersionKey: req.GroupID, @@ -74,14 +76,20 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { - if elem.EID == "" { + logs := make([]model.VersionLogElem, 0, len(vl.Logs)) + for i, log := range vl.Logs { + switch log.EID { + case model.VersionGroupChangeID: vl.LogLen-- hasGroupUpdate = true - return true + case model.VersionSortChangeID: + vl.LogLen-- + sortVersion = uint64(log.Version) + default: + logs = append(logs, vl.Logs[i]) } - return false - }) + } + vl.Logs = logs if vl.LogLen > 0 { hasGroupUpdate = true } @@ -94,12 +102,13 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - Delete: delIDs, - Insert: insertList, - Update: updateList, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, } }, } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 8de21e83a5..42b3dd72bd 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -118,7 +118,7 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": bson.M{"role_level": roleLevel}}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, userID}, model.VersionStateUpdate) }) } func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error { @@ -131,10 +131,9 @@ func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID strin bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil { return err } - return nil }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{firstUserID, secondUserID}, model.VersionStateUpdate) + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, firstUserID, secondUserID}, model.VersionStateUpdate) }) } @@ -145,7 +144,13 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + var userIDs []string + if g.IsUpdateRoleLevel(data) { + userIDs = []string{model.VersionSortChangeID, userID} + } else { + userIDs = []string{userID} + } + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 8836742f08..80b5658c6c 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -152,8 +152,24 @@ func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []s "$unset": "delete_e_ids", }, } - opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0}) - return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) + projection := bson.M{ + "logs": 0, + } + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(projection) + res, err := mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) + if err != nil { + return nil, err + } + res.Logs = make([]model.VersionLogElem, 0, len(eIds)) + for _, id := range eIds { + res.Logs = append(res.Logs, model.VersionLogElem{ + EID: id, + State: state, + Version: res.Version, + LastUpdate: res.LastUpdate, + }) + } + return res, nil } func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go index 236c61a2ce..4576e45bcd 100644 --- a/pkg/common/storage/database/mgo/version_test.go +++ b/pkg/common/storage/database/mgo/version_test.go @@ -9,12 +9,12 @@ import ( "time" ) -func Result[V any](val V, err error) V { - if err != nil { - panic(err) - } - return val -} +//func Result[V any](val V, err error) V { +// if err != nil { +// panic(err) +// } +// return val +//} func Check(err error) { if err != nil { @@ -30,7 +30,7 @@ func TestName(t *testing.T) { panic(err) } vl := tmp.(*VersionLogMgo) - res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) + res, err := vl.incrVersionResult(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert) if err != nil { t.Log(err) return diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index 11a40ef24f..2091a3a293 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -14,6 +14,11 @@ const ( VersionStateUpdate ) +const ( + VersionGroupChangeID = "" + VersionSortChangeID = "$" +) + type VersionLogElem struct { EID string `bson:"e_id"` State int32 `bson:"state"` From 1ef057bbe20f6cbd4689410fdd50feccb8a6edbc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 17:24:52 +0800 Subject: [PATCH 130/147] sort version --- internal/rpc/group/sync.go | 5 +++++ pkg/common/storage/database/mgo/friend.go | 26 ++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 12598510be..f89a98ee84 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -12,6 +12,11 @@ import ( "github.com/openimsdk/protocol/sdkws" ) +func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (*pbgroup.BatchGetIncrementalGroupMemberResp, error) { + //TODO implement me + panic("implement me") +} + func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 7f456fbdab..76c82bac24 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -109,7 +109,13 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate) + var friendUserIDs []string + if f.IsUpdateIsPinned(args) { + friendUserIDs = []string{model.VersionSortChangeID, friendUserID} + } else { + friendUserIDs = []string{friendUserID} + } + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) }) } @@ -214,7 +220,7 @@ func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ( func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { // Ensure there are IDs to update - if len(friendUserIDs) == 0 { + if len(friendUserIDs) == 0 || len(val) == 0 { return nil // Or return an error if you expect there to always be IDs } @@ -230,7 +236,13 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) + var userIDs []string + if f.IsUpdateIsPinned(val) { + userIDs = append([]string{model.VersionSortChangeID}, friendUserIDs...) + } else { + userIDs = friendUserIDs + } + return f.owner.IncrVersion(ctx, ownerUserID, userIDs, model.VersionStateUpdate) }) } @@ -248,3 +260,11 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) } + +func (f *FriendMgo) IsUpdateIsPinned(data map[string]any) bool { + if data == nil { + return false + } + _, ok := data["is_pinned"] + return ok +} From ca1b1a2d723e4c3b92932e380e1c263823dad59d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 17:37:08 +0800 Subject: [PATCH 131/147] sort version --- pkg/common/storage/model/version_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index 2091a3a293..6ed8d30f2e 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -16,7 +16,7 @@ const ( const ( VersionGroupChangeID = "" - VersionSortChangeID = "$" + VersionSortChangeID = "____S_O_R_T_I_D____" ) type VersionLogElem struct { From ddf8f0d5cd7884df114e4506fbafe807b0443df9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 17:55:33 +0800 Subject: [PATCH 132/147] remove old version online subscription --- internal/rpc/user/online.go | 42 +---- internal/rpc/user/user.go | 3 +- pkg/common/storage/controller/user.go | 49 +---- pkg/common/storage/database/mgo/subscribe.go | 189 ------------------- pkg/common/storage/database/subscribe.go | 31 --- 5 files changed, 7 insertions(+), 307 deletions(-) delete mode 100644 pkg/common/storage/database/mgo/subscribe.go delete mode 100644 pkg/common/storage/database/subscribe.go diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go index e853ceae23..99b272006f 100644 --- a/internal/rpc/user/online.go +++ b/internal/rpc/user/online.go @@ -3,7 +3,6 @@ package user import ( "context" "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/sdkws" pbuser "github.com/openimsdk/protocol/user" ) @@ -38,23 +37,6 @@ func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) // SubscribeOrCancelUsersStatus Subscribe online or cancel online users. func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { - if req.Genre == constant.SubscriberUser { - err := s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.getUsersOnlineStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err := s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil } @@ -82,34 +64,12 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { return nil, err } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - return &pbuser.SetUserStatusResp{}, nil } // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.getUsersOnlineStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil + return &pbuser.GetSubscribeUsersStatusResp{}, nil } func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 0b96077ec3..779d9b0c4d 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -93,8 +93,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi return err } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) - userMongoDB := mgo.NewUserMongoDriver(mgocli.GetDB()) - database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx(), userMongoDB) + database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 59559537b1..5ce8104e7b 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -62,14 +62,6 @@ type UserDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) - // SubscribeUsersStatus Subscribe a user's presence status - SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // UnsubscribeUsersStatus unsubscribe a user's presence status - UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // GetAllSubscribeList Get a list of all subscriptions - GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) - // GetSubscribedList Get all subscribed lists - GetSubscribedList(ctx context.Context, userID string) ([]string, error) // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -80,14 +72,13 @@ type UserDatabase interface { } type userDatabase struct { - tx tx.Tx - userDB database.User - cache cache.UserCache - mongoDB database.SubscribeUser + tx tx.Tx + userDB database.User + cache cache.UserCache } -func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx, mongoDB database.SubscribeUser) UserDatabase { - return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB} +func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx) UserDatabase { + return &userDatabase{userDB: userDB, cache: cache, tx: tx} } func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error { @@ -212,36 +203,6 @@ func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]stri return u.userDB.SortQuery(ctx, userIDName, asc) } -// SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. -func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) - return err -} - -// UnsubscribeUsersStatus unsubscribe a user's presence status. -func (u *userDatabase) UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.UnsubscriptionList(ctx, userID, userIDs) - return err -} - -// GetAllSubscribeList Get a list of all subscriptions. -func (u *userDatabase) GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetAllSubscribeList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - -// GetSubscribedList Get all subscribed lists. -func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetSubscribedList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex) } diff --git a/pkg/common/storage/database/mgo/subscribe.go b/pkg/common/storage/database/mgo/subscribe.go deleted file mode 100644 index 5b7d9786b7..0000000000 --- a/pkg/common/storage/database/mgo/subscribe.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - - "github.com/openimsdk/tools/errs" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -// prefixes and suffixes. -const ( - SubscriptionPrefix = "subscription_prefix" - SubscribedPrefix = "subscribed_prefix" -) - -// MaximumSubscription Maximum number of subscriptions. -const ( - MaximumSubscription = 3000 -) - -func NewUserMongoDriver(database *mongo.Database) database.SubscribeUser { - return &UserMongoDriver{ - userCollection: database.Collection(model.SubscribeUserTableName), - } -} - -type UserMongoDriver struct { - userCollection *mongo.Collection -} - -// AddSubscriptionList Subscriber's handling of thresholds. -func (u *UserMongoDriver) AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error { - // Check the number of lists in the key. - pipeline := mongo.Pipeline{ - {{"$match", bson.D{{"user_id", SubscriptionPrefix + userID}}}}, - {{"$project", bson.D{{"count", bson.D{{"$size", "$user_id_list"}}}}}}, - } - // perform aggregate operations - cursor, err := u.userCollection.Aggregate(ctx, pipeline) - if err != nil { - return errs.Wrap(err) - } - defer cursor.Close(ctx) - var cnt struct { - Count int `bson:"count"` - } - // iterate over aggregated results - for cursor.Next(ctx) { - err = cursor.Decode(&cnt) - if err != nil { - return errs.Wrap(err) - } - } - var newUserIDList []string - // If the threshold is exceeded, pop out the previous MaximumSubscription - len(userIDList) and insert it. - if cnt.Count+len(userIDList) > MaximumSubscription { - newUserIDList, err = u.GetAllSubscribeList(ctx, userID) - if err != nil { - return err - } - newUserIDList = newUserIDList[MaximumSubscription-len(userIDList):] - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$set": bson.M{"user_id_list": newUserIDList}}, - ) - if err != nil { - return err - } - // Another way to subscribe to N before pop,Delete after testing - /*for i := 1; i <= MaximumSubscription-len(userIDList); i++ { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{SubscriptionPrefix + userID: bson.M{"$pop": -1}}, - ) - if err != nil { - return err - } - }*/ - } - upsert := true - opts := &options.UpdateOptions{ - Upsert: &upsert, - } - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$addToSet": bson.M{"user_id_list": bson.M{"$each": userIDList}}}, - opts, - ) - if err != nil { - return errs.Wrap(err) - } - for _, user := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + user}, - bson.M{"$addToSet": bson.M{"user_id_list": userID}}, - opts, - ) - if err != nil { - return errs.WrapMsg(err, "transaction failed") - } - } - return nil -} - -// UnsubscriptionList Handling of unsubscribe. -func (u *UserMongoDriver) UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$pull": bson.M{"user_id_list": bson.M{"$in": userIDList}}}, - ) - if err != nil { - return errs.Wrap(err) - } - err = u.RemoveSubscribedListFromUser(ctx, userID, userIDList) - if err != nil { - return errs.Wrap(err) - } - return nil -} - -// RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. -func (u *UserMongoDriver) RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error { - var err error - for _, userIDTemp := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userIDTemp}, - bson.M{"$pull": bson.M{"user_id_list": userID}}, - ) - } - return errs.Wrap(err) -} - -// GetAllSubscribeList Get all users subscribed by this user. -func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} - -// GetSubscribedList Get the user subscribed by those users. -func (u *UserMongoDriver) GetSubscribedList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} diff --git a/pkg/common/storage/database/subscribe.go b/pkg/common/storage/database/subscribe.go deleted file mode 100644 index 5905ecd073..0000000000 --- a/pkg/common/storage/database/subscribe.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import "context" - -// SubscribeUser Operation interface of user mongodb. -type SubscribeUser interface { - // AddSubscriptionList Subscriber's handling of thresholds. - AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error - // UnsubscriptionList Handling of unsubscribe. - UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error - // RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. - RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error - // GetAllSubscribeList Get all users subscribed by this user - GetAllSubscribeList(ctx context.Context, id string) (userIDList []string, err error) - // GetSubscribedList Get the user subscribed by those users - GetSubscribedList(ctx context.Context, id string) (userIDList []string, err error) -} From f8394d4fd87f7db482f4c7c575a4e07b41235e32 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 18:05:09 +0800 Subject: [PATCH 133/147] remove old version online subscription --- tools/seq/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/seq/main.go b/tools/seq/main.go index 399e6d934e..16da9f156e 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -12,7 +12,7 @@ func main() { config string second int ) - flag.StringVar(&config, "c", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + flag.StringVar(&config, "c", "", "config directory") flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { From 187903e3c3a60ecc9eefc480da48012af718ab2b Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 18 Jul 2024 18:10:54 +0800 Subject: [PATCH 134/147] version log index --- pkg/common/storage/database/mgo/version_log.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 80b5658c6c..3b449007bc 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -18,8 +18,8 @@ import ( func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { lm := &VersionLogMgo{coll: coll} - if lm.initIndex(context.Background()) != nil { - return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) + if err := lm.initIndex(context.Background()); err != nil { + return nil, errs.WrapMsg(err, "init version log index failed", "coll", coll.Name()) } return lm, nil } @@ -33,6 +33,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { Keys: bson.M{ "d_id": 1, }, + Options: options.Index().SetUnique(true), }) return err } From 276c0ab1886de749b06fb947c5143f9374e05b9d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 22 Jul 2024 17:19:25 +0800 Subject: [PATCH 135/147] version log index --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/hub_server.go | 179 +++++++++++++++++++++++------- 3 files changed, 142 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 0637492eb5..fa40effddd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.38 - github.com/openimsdk/tools v0.0.49-alpha.51 + github.com/openimsdk/tools v0.0.49-alpha.52 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 92a783c069..cc0f5f7664 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.38 h1:kVZCHIXg/el8YJFoIBWhZu1sbbTUqmzgF4l0W3sUH24= github.com/openimsdk/protocol v0.0.69-alpha.38/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.51 h1:JTPEetVSNOczw1n+XjiPozaH2SBPQAc+9VlPE41wEeY= -github.com/openimsdk/tools v0.0.49-alpha.51/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/tools v0.0.49-alpha.52 h1:NwAAtBO4BV96qG6Z0P2btGEqn4AI2DFgaHvLMXNHal0= +github.com/openimsdk/tools v0.0.49-alpha.52/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 3891aa532c..33e761a22c 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -22,11 +22,15 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mq/memamq" + "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" + "sync/atomic" ) func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { @@ -57,6 +61,7 @@ type Server struct { pushTerminal map[int]struct{} ready func(srv *Server) error userRcp rpcclient.UserRpcClient + queue *memamq.MemoryQueue } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { @@ -70,6 +75,7 @@ func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready f pushTerminal: make(map[int]struct{}), config: conf, ready: ready, + queue: memamq.NewMemoryQueue(512, 1024*16), } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} @@ -125,57 +131,150 @@ func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.Onli return nil, nil } -func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq, -) (*msggateway.OnlineBatchPushOneMsgResp, error) { - var singleUserResults []*msggateway.SingleMsgToUserResults - for _, v := range req.PushToUserIDs { - var resp []*msggateway.SingleMsgToUserPlatform - results := &msggateway.SingleMsgToUserResults{ - UserID: v, +func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { + clients, ok := s.LongConnServer.GetUserAllCons(userID) + if !ok { + log.ZDebug(ctx, "push user not online", "userID", userID) + return &msggateway.SingleMsgToUserResults{ + UserID: userID, } - clients, ok := s.LongConnServer.GetUserAllCons(v) - if !ok { - log.ZDebug(ctx, "push user not online", "userID", v) - results.Resp = resp - singleUserResults = append(singleUserResults, results) + } + log.ZDebug(ctx, "push user online", "clients", clients, "userID", userID) + result := &msggateway.SingleMsgToUserResults{ + UserID: userID, + Resp: make([]*msggateway.SingleMsgToUserPlatform, 0, len(clients)), + } + for _, client := range clients { + if client == nil { continue } - - log.ZDebug(ctx, "push user online", "clients", clients, "userID", v) - for _, client := range clients { - if client == nil { - continue + userPlatform := &msggateway.SingleMsgToUserPlatform{ + RecvPlatFormID: int32(client.PlatformID), + } + if !client.IsBackground || + (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { + err := client.PushMessage(ctx, msgData) + if err != nil { + userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) + //result.Resp = append(result.Resp, userPlatform) + //resp = append(resp, userPlatform) + } else { + if _, ok := s.pushTerminal[client.PlatformID]; ok { + result.OnlinePush = true + //result.Resp = append(result.Resp, userPlatform) + //results.OnlinePush = true + //resp = append(resp, userPlatform) + } } + } else { + userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) + //resp = append(resp, userPlatform) + } + result.Resp = append(result.Resp, userPlatform) + } + return result +} - userPlatform := &msggateway.SingleMsgToUserPlatform{ - RecvPlatFormID: int32(client.PlatformID), +func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { + if len(req.PushToUserIDs) == 0 { + return &msggateway.OnlineBatchPushOneMsgResp{}, nil + } + ch := make(chan *msggateway.SingleMsgToUserResults, len(req.PushToUserIDs)) + var count atomic.Int64 + count.Add(int64(len(req.PushToUserIDs))) + for _, userID := range req.PushToUserIDs { + err := s.queue.PushCtx(ctx, func() { + ch <- s.pushToUser(ctx, userID, req.MsgData) + if count.Add(-1) == 0 { + close(ch) } - if !client.IsBackground || - (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { - err := client.PushMessage(ctx, req.MsgData) - if err != nil { - userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) - resp = append(resp, userPlatform) - } else { - if _, ok := s.pushTerminal[client.PlatformID]; ok { - results.OnlinePush = true - resp = append(resp, userPlatform) - } - } - } else { - userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) - resp = append(resp, userPlatform) + }) + if err != nil { + if count.Add(-1) == 0 { + close(ch) + } + log.ZError(ctx, "pushToUser MemoryQueue failed", err, "userID", userID) + ch <- &msggateway.SingleMsgToUserResults{ + UserID: userID, } } - results.Resp = resp - singleUserResults = append(singleUserResults, results) } - - return &msggateway.OnlineBatchPushOneMsgResp{ - SinglePushResult: singleUserResults, - }, nil + resp := &msggateway.OnlineBatchPushOneMsgResp{ + SinglePushResult: make([]*msggateway.SingleMsgToUserResults, 0, len(req.PushToUserIDs)), + } + for { + select { + case <-ctx.Done(): + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ctx done", context.Cause(ctx)) + userIDSet := datautil.SliceSet(req.PushToUserIDs) + for _, results := range resp.SinglePushResult { + delete(userIDSet, results.UserID) + } + for userID := range userIDSet { + resp.SinglePushResult = append(resp.SinglePushResult, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return resp, nil + case res, ok := <-ch: + if !ok { + return resp, nil + } + resp.SinglePushResult = append(resp.SinglePushResult, res) + } + } } +//func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { +// var singleUserResults []*msggateway.SingleMsgToUserResults +// for _, userID := range req.PushToUserIDs { +// var resp []*msggateway.SingleMsgToUserPlatform +// results := &msggateway.SingleMsgToUserResults{ +// UserID: userID, +// } +// clients, ok := s.LongConnServer.GetUserAllCons(userID) +// if !ok { +// log.ZDebug(ctx, "push user not online", "userID", userID) +// results.Resp = resp +// singleUserResults = append(singleUserResults, results) +// continue +// } +// +// log.ZDebug(ctx, "push user online", "clients", clients, "userID", userID) +// for _, client := range clients { +// if client == nil { +// continue +// } +// +// userPlatform := &msggateway.SingleMsgToUserPlatform{ +// RecvPlatFormID: int32(client.PlatformID), +// } +// if !client.IsBackground || +// (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { +// err := client.PushMessage(ctx, req.MsgData) +// if err != nil { +// userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) +// resp = append(resp, userPlatform) +// } else { +// if _, ok := s.pushTerminal[client.PlatformID]; ok { +// results.OnlinePush = true +// resp = append(resp, userPlatform) +// } +// } +// } else { +// userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) +// resp = append(resp, userPlatform) +// } +// } +// results.Resp = resp +// singleUserResults = append(singleUserResults, results) +// } +// +// return &msggateway.OnlineBatchPushOneMsgResp{ +// SinglePushResult: singleUserResults, +// }, nil +//} + func (s *Server) KickUserOffline( ctx context.Context, req *msggateway.KickUserOfflineReq, From 48db156ff7b51401088550c89df342ec5b2e30bd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 22 Jul 2024 17:39:50 +0800 Subject: [PATCH 136/147] batch push --- internal/msggateway/hub_server.go | 53 ++----------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 33e761a22c..3d678d0c24 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -182,7 +182,8 @@ func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msgga ch := make(chan *msggateway.SingleMsgToUserResults, len(req.PushToUserIDs)) var count atomic.Int64 count.Add(int64(len(req.PushToUserIDs))) - for _, userID := range req.PushToUserIDs { + for i := range req.PushToUserIDs { + userID := req.PushToUserIDs[i] err := s.queue.PushCtx(ctx, func() { ch <- s.pushToUser(ctx, userID, req.MsgData) if count.Add(-1) == 0 { @@ -225,56 +226,6 @@ func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msgga } } -//func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { -// var singleUserResults []*msggateway.SingleMsgToUserResults -// for _, userID := range req.PushToUserIDs { -// var resp []*msggateway.SingleMsgToUserPlatform -// results := &msggateway.SingleMsgToUserResults{ -// UserID: userID, -// } -// clients, ok := s.LongConnServer.GetUserAllCons(userID) -// if !ok { -// log.ZDebug(ctx, "push user not online", "userID", userID) -// results.Resp = resp -// singleUserResults = append(singleUserResults, results) -// continue -// } -// -// log.ZDebug(ctx, "push user online", "clients", clients, "userID", userID) -// for _, client := range clients { -// if client == nil { -// continue -// } -// -// userPlatform := &msggateway.SingleMsgToUserPlatform{ -// RecvPlatFormID: int32(client.PlatformID), -// } -// if !client.IsBackground || -// (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { -// err := client.PushMessage(ctx, req.MsgData) -// if err != nil { -// userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) -// resp = append(resp, userPlatform) -// } else { -// if _, ok := s.pushTerminal[client.PlatformID]; ok { -// results.OnlinePush = true -// resp = append(resp, userPlatform) -// } -// } -// } else { -// userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) -// resp = append(resp, userPlatform) -// } -// } -// results.Resp = resp -// singleUserResults = append(singleUserResults, results) -// } -// -// return &msggateway.OnlineBatchPushOneMsgResp{ -// SinglePushResult: singleUserResults, -// }, nil -//} - func (s *Server) KickUserOffline( ctx context.Context, req *msggateway.KickUserOfflineReq, From 2e7a4e03d2879177106119f81fb6fd81c90ba4e1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 22 Jul 2024 17:48:43 +0800 Subject: [PATCH 137/147] batch push --- internal/msggateway/hub_server.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 3d678d0c24..28c227162a 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -156,19 +156,13 @@ func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.M err := client.PushMessage(ctx, msgData) if err != nil { userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) - //result.Resp = append(result.Resp, userPlatform) - //resp = append(resp, userPlatform) } else { if _, ok := s.pushTerminal[client.PlatformID]; ok { result.OnlinePush = true - //result.Resp = append(result.Resp, userPlatform) - //results.OnlinePush = true - //resp = append(resp, userPlatform) } } } else { userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) - //resp = append(resp, userPlatform) } result.Resp = append(result.Resp, userPlatform) } From c9c961e92754a84f3e0569c5121ca2e3957a1269 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 24 Jul 2024 14:37:21 +0800 Subject: [PATCH 138/147] seq void filling --- pkg/common/storage/database/mgo/msg.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index ebabadf52c..2d1819c3a6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -94,6 +94,29 @@ func (m *MsgMgo) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDo } func (m *MsgMgo) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { + msgs, err := m.getMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs) + if err != nil { + return nil, err + } + if len(msgs) == len(seqs) { + return msgs, nil + } + tmp := make(map[int64]*model.MsgInfoModel) + for i, val := range msgs { + tmp[val.Msg.Seq] = msgs[i] + } + res := make([]*model.MsgInfoModel, 0, len(seqs)) + for _, seq := range seqs { + if val, ok := tmp[seq]; ok { + res = append(res, val) + } else { + res = append(res, &model.MsgInfoModel{Msg: &model.MsgDataModel{Seq: seq}}) + } + } + return res, nil +} + +func (m *MsgMgo) getMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { indexs := make([]int64, 0, len(seqs)) for _, seq := range seqs { indexs = append(indexs, m.model.GetMsgIndex(seq)) From 130543301ddadddbc3dd071a4aaa05411aaa27de Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 24 Jul 2024 14:37:30 +0800 Subject: [PATCH 139/147] fix: batchGetMaxSeq --- pkg/common/storage/cache/redis/seq_conversation.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index fb8a547dfa..7e75bff3ff 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -78,12 +78,14 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s } } if len(notFoundKey) > 0 { - conversationID := keyConversationID[notFoundKey[0]] - seq, err := s.GetMaxSeq(ctx, conversationID) - if err != nil { - return err + for _, key := range notFoundKey { + conversationID := keyConversationID[key] + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return err + } + seqs[conversationID] = seq } - seqs[conversationID] = seq } return nil } From 61fc375ce86c02f5577ac61ce7ead3175a7c20b1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 24 Jul 2024 14:41:50 +0800 Subject: [PATCH 140/147] fix: batchGetMaxSeq --- pkg/common/storage/cache/redis/seq_conversation.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index 7e75bff3ff..7fe849193e 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -77,15 +77,13 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s return errs.Wrap(err) } } - if len(notFoundKey) > 0 { - for _, key := range notFoundKey { - conversationID := keyConversationID[key] - seq, err := s.GetMaxSeq(ctx, conversationID) - if err != nil { - return err - } - seqs[conversationID] = seq + for _, key := range notFoundKey { + conversationID := keyConversationID[key] + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return err } + seqs[conversationID] = seq } return nil } From 17bd885112e08599472445e8cf67f0c6cf68112d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 25 Jul 2024 15:11:14 +0800 Subject: [PATCH 141/147] cache db error log --- pkg/common/storage/cache/redis/batch.go | 2 ++ pkg/common/storage/cache/redis/batch_handler.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 5f9a8c82db..4d65c59298 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "golang.org/x/sync/singleflight" "time" @@ -49,6 +50,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } values, err := fn(ctx, queryIds) if err != nil { + log.ZError(ctx, "batchGetCache query database failed", err, "keys", keys, "queryIds", queryIds) return nil, err } if len(values) == 0 { diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 52e046a408..f9923e198e 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -118,6 +118,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { + log.ZError(ctx, "getCache query database failed", err, "key", key) return "", err } bs, err := json.Marshal(t) From a04e0e8ec2a898b1ed896f484ae71ea61df1d7da Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 25 Jul 2024 19:15:26 +0800 Subject: [PATCH 142/147] 111 --- pkg/common/storage/cache/redis/seq_user.go | 40 ++++++++++----------- pkg/common/storage/cache/seq_user.go | 18 +++++----- pkg/common/storage/controller/msg.go | 20 +++++------ pkg/common/storage/database/mgo/seq_user.go | 14 ++++---- pkg/common/storage/database/seq_user.go | 14 ++++---- tools/seq/internal/main.go | 8 ++--- 6 files changed, 57 insertions(+), 57 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index 2ad43eebdb..edbc66b21b 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -44,38 +44,38 @@ func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID s return cachekey.GetSeqUserReadSeqKey(conversationID, userID) } -func (s *seqUserCacheRedis) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserMaxSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - if err := s.mgo.SetMaxSeq(ctx, conversationID, userID, seq); err != nil { +func (s *seqUserCacheRedis) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetUserMaxSeq(ctx, conversationID, userID, seq); err != nil { return err } return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) } -func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserMinSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - return s.SetMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) +func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) } -func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserReadSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -85,10 +85,10 @@ func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID strin return nil } -func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { keys := make([]string, 0, len(seqs)) for conversationID, seq := range seqs { - if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserMinSeq(ctx, conversationID, userID, seq); err != nil { return err } keys = append(keys, s.getSeqUserMinSeqKey(conversationID, userID)) @@ -96,7 +96,7 @@ func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs return DeleteCacheBySlot(ctx, s.rocks, keys) } -func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { keys := make([]string, 0, len(seqs)) keySeq := make(map[string]int64) for conversationID, seq := range seqs { @@ -121,16 +121,16 @@ func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, return nil } -func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { if len(seqs) == 0 { return nil } - if err := s.setRedisReadSeqs(ctx, userID, seqs); err != nil { + if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil { return err } for conversationID, seq := range seqs { if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -138,13 +138,13 @@ func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs return nil } -func (s *seqUserCacheRedis) GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { +func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { return s.getSeqUserReadSeqKey(conversationID, userID) }, func(v *readSeqModel) string { return v.ConversationID }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { - seqs, err := s.mgo.GetReadSeqs(ctx, userID, conversationIDs) + seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs) if err != nil { return nil, err } diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go index 4d0bb4ffab..61dbc0ab45 100644 --- a/pkg/common/storage/cache/seq_user.go +++ b/pkg/common/storage/cache/seq_user.go @@ -3,13 +3,13 @@ package cache import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error - SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error - GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error + SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 32202ac9e9..a6f0dbbb04 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -334,7 +334,7 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -498,7 +498,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -576,7 +576,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -674,12 +674,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -794,23 +794,23 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int } func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seqUser.SetMinSeqs(ctx, userID, seqs) + return db.seqUser.SetUserMinSeqs(ctx, userID, seqs) } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seqUser.SetReadSeqs(ctx, userID, hasReadSeqs) + return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetReadSeq(ctx, conversationID, userID, hasReadSeq) + return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seqUser.GetReadSeqs(ctx, userID, conversationIDs) + return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seqUser.GetReadSeq(ctx, conversationID, userID) + return db.seqUser.GetUserReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index e0cbb08d9c..7c9a8f1336 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -68,27 +68,27 @@ func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID } } -func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "max_seq") } -func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "max_seq") } -func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "min_seq") } -func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "min_seq") } -func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "read_seq") } -func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { +func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { if len(conversationID) == 0 { return map[string]int64{}, nil } @@ -105,6 +105,6 @@ func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversat return res, nil } -func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go index edd3910d02..9f75c710b6 100644 --- a/pkg/common/storage/database/seq_user.go +++ b/pkg/common/storage/database/seq_user.go @@ -3,11 +3,11 @@ package database import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) } diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 7f021a90e8..2bec5a8f1c 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -130,14 +130,14 @@ func Main(conf string, del time.Duration) error { if err != nil { return 0, err } - return uSeq.GetReadSeq(ctx, conversationID, userID) + return uSeq.GetUserReadSeq(ctx, conversationID, userID) }, SetSeq: func(ctx context.Context, id string, seq int64) error { conversationID, userID, err := uSpitHasReadSeq(id) if err != nil { return err } - return uSeq.SetReadSeq(ctx, conversationID, userID, seq) + return uSeq.SetUserReadSeq(ctx, conversationID, userID, seq) }, }, { @@ -147,14 +147,14 @@ func Main(conf string, del time.Duration) error { if err != nil { return 0, err } - return uSeq.GetMinSeq(ctx, conversationID, userID) + return uSeq.GetUserMinSeq(ctx, conversationID, userID) }, SetSeq: func(ctx context.Context, id string, seq int64) error { conversationID, userID, err := uSpitConversationUserMinSeq(id) if err != nil { return err } - return uSeq.SetMinSeq(ctx, conversationID, userID, seq) + return uSeq.SetUserMinSeq(ctx, conversationID, userID, seq) }, }, } From 60eb5e913eda26b0b084e2eabc830221899f911a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 26 Jul 2024 15:16:04 +0800 Subject: [PATCH 143/147] fix bug --- internal/rpc/friend/friend.go | 4 +- internal/rpc/friend/sync.go | 22 +- internal/rpc/msg/sync_msg.go | 4 +- pkg/common/storage/cache/redis/batch_test.go | 2 +- .../storage/cache/redis/seq_user_test.go | 32 ++ pkg/common/storage/controller/msg.go | 4 +- pkg/common/storage/database/mgo/msg.go | 457 ++++++++++++++---- pkg/common/storage/database/mgo/msg_test.go | 75 +++ .../database/mgo/seq_conversation_test.go | 7 +- pkg/common/storage/database/mgo/seq_user.go | 9 + pkg/common/storage/database/msg.go | 2 +- 11 files changed, 518 insertions(+), 100 deletions(-) create mode 100644 pkg/common/storage/database/mgo/msg_test.go diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 622e19f422..bdb786bca4 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -49,6 +50,7 @@ type friendServer struct { RegisterCenter discovery.SvcDiscoveryRegistry config *Config webhookClient *webhook.Client + queue *memamq.MemoryQueue } type Config struct { @@ -118,8 +120,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + queue: memamq.NewMemoryQueue(128, 1024*8), }) - return nil } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 145c287da0..902cc73037 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" "slices" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" @@ -17,14 +18,23 @@ func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *rela if err != nil { return nil, err } - for _, userID := range userIDs { - if err := s.db.OwnerIncrVersion(ctx, userID, []string{req.UserID}, model.VersionStateUpdate); err != nil { - return nil, err + if len(userIDs) > 0 { + friendUserIDs := []string{req.UserID} + noCancelCtx := context.WithoutCancel(ctx) + err := s.queue.PushCtx(ctx, func() { + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(noCancelCtx, userID, friendUserIDs, model.VersionStateUpdate); err != nil { + log.ZError(ctx, "OwnerIncrVersion", err, "userID", userID, "friendUserIDs", friendUserIDs) + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(noCancelCtx, req.UserID, userID) + } + }) + if err != nil { + log.ZError(ctx, "NotificationUserInfoUpdate timeout", err, "userID", req.UserID) } } - for _, userID := range userIDs { - s.notificationSender.FriendInfoUpdatedNotification(ctx, req.UserID, userID) - } return &relation.NotificationUserInfoUpdateResp{}, nil } diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index afb79506ee..f5b5ebda53 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -111,7 +111,7 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (resp *msg.SearchMessageResp, err error) { var chatLogs []*sdkws.MsgData - var total int32 + var total int64 resp = &msg.SearchMessageResp{} if total, chatLogs, err = m.MsgDatabase.SearchMessage(ctx, req); err != nil { return nil, err @@ -194,7 +194,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq } resp.ChatLogs = append(resp.ChatLogs, pbchatLog) } - resp.ChatLogsNum = total + resp.ChatLogsNum = int32(total) return resp, nil } diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go index e4caa2a21f..bbb6d76f15 100644 --- a/pkg/common/storage/cache/redis/batch_test.go +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -45,7 +45,7 @@ func TestName(t *testing.T) { } seqUser := NewSeqUserCacheRedis(rdb, mgoSeqUser) - res, err := seqUser.GetReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) + res, err := seqUser.GetUserReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) if err != nil { panic(err) } diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index e4fd95922a..0059c81db9 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + mgo2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "log" "strconv" "sync/atomic" @@ -77,3 +80,32 @@ func TestRecvOnline(t *testing.T) { fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) } } + +func TestName1(t *testing.T) { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 0, + } + rdb := redis.NewClient(opt) + + mgo, err := mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo2.NewSeqUserMongo(mgo.Database("openim_v3")) + if err != nil { + panic(err) + } + seq := NewSeqUserCacheRedis(rdb, model) + + res, err := seq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"}) + if err != nil { + panic(err) + } + t.Log(res) + +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a6f0dbbb04..4ea74ef690 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -84,7 +84,7 @@ type CommonMsgDatabase interface { //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) + SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) FindOneByDocIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) // to mq @@ -878,7 +878,7 @@ func (db *commonMsgDatabase) RangeGroupSendCount( return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) { +func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) { var totalMsgs []*sdkws.MsgData total, msgs, err := db.msgDocDatabase.SearchMessage(ctx, req) if err != nil { diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 2d1819c3a6..03f47c5033 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -278,124 +278,409 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do return nil } -func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { - where := make(bson.A, 0, 6) +//func (m *MsgMgo) searchCount(ctx context.Context, filter any) (int64, error) { +// +// return nil, nil +//} + +//func (m *MsgMgo) searchMessage(ctx context.Context, filter any, nextID primitive.ObjectID, content bool, limit int) (int64, []*model.MsgInfoModel, primitive.ObjectID, error) { +// var pipeline bson.A +// if !nextID.IsZero() { +// pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) +// } +// pipeline = append(pipeline, +// bson.M{"$match": filter}, +// bson.M{"$limit": limit}, +// bson.M{"$unwind": "$msgs"}, +// bson.M{"$match": filter}, +// bson.M{ +// "$group": bson.M{ +// "_id": "$_id", +// "doc_id": bson.M{ +// "$first": "$doc_id", +// }, +// "msgs": bson.M{"$push": "$msgs"}, +// }, +// }, +// ) +// if !content { +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 1, +// "count": bson.M{"$size": "$msgs"}, +// }, +// }, +// ) +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Count int64 `bson:"count"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, nil +// } +// var count int64 +// for _, r := range res { +// count += r.Count +// } +// return count, nil, res[len(res)-1].ID, nil +// } +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Msg []*model.MsgInfoModel `bson:"msgs"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, err +// } +// var count int +// for _, r := range res { +// count += len(r.Msg) +// } +// msgs := make([]*model.MsgInfoModel, 0, count) +// for _, r := range res { +// msgs = append(msgs, r.Msg...) +// } +// return int64(count), msgs, res[len(res)-1].ID, nil +//} + +/* + +db.msg3.aggregate( + [ + { + "$match": { + "doc_id": "si_7009965934_8710838466:0" + }, + + } + ] +) + + +*/ + +type searchMessageIndex struct { + ID primitive.ObjectID `bson:"_id"` + Index []int64 `bson:"index"` +} + +func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID primitive.ObjectID, limit int) ([]searchMessageIndex, error) { + var pipeline bson.A + if !nextID.IsZero() { + pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) + } + pipeline = append(pipeline, + bson.M{"$sort": bson.M{"_id": 1}}, + bson.M{"$match": filter}, + bson.M{"$limit": limit}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs": bson.M{ + "$map": bson.M{ + "input": "$msgs", + "as": "msg", + "in": bson.M{ + "$mergeObjects": bson.A{ + "$$msg", + bson.M{ + "_search_temp_index": bson.M{ + "$indexOfArray": bson.A{ + "$msgs", "$$msg", + }, + }, + }, + }, + }, + }, + }, + }, + }, + bson.M{"$unwind": "$msgs"}, + bson.M{"$match": filter}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs._search_temp_index": 1, + }, + }, + bson.M{ + "$group": bson.M{ + "_id": "$_id", + "index": bson.M{"$push": "$msgs._search_temp_index"}, + }, + }, + bson.M{"$sort": bson.M{"_id": 1}}, + ) + return mongoutil.Aggregate[searchMessageIndex](ctx, m.coll, pipeline) +} + +func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []searchMessageIndex, error) { + filter := bson.M{} if req.RecvID != "" { - where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) + filter["$or"] = bson.A{ + bson.M{"msgs.msg.recv_id": req.RecvID}, + bson.M{"msgs.msg.group_id": req.RecvID}, + } } if req.SendID != "" { - where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) + filter["msgs.msg.send_id"] = req.SendID } if req.ContentType != 0 { - where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) + filter["msgs.msg.content_type"] = req.ContentType } if req.SessionType != 0 { - where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) + filter["msgs.msg.session_type"] = req.SessionType } if req.SendTime != "" { sendTime, err := time.Parse(time.DateOnly, req.SendTime) if err != nil { return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) } - where = append(where, - bson.M{ - "msgs.msg.send_time": bson.M{ - "$gte": sendTime.UnixMilli(), - }, - }, + filter["$and"] = bson.A{ + bson.M{"msgs.msg.send_time": bson.M{ + "$gte": sendTime.UnixMilli(), + }}, bson.M{ "msgs.msg.send_time": bson.M{ "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), }, }, - ) - } - pipeline := bson.A{ - bson.M{ - "$unwind": "$msgs", - }, - } - if len(where) > 0 { - pipeline = append(pipeline, bson.M{ - "$match": bson.M{"$and": where}, - }) + } } - pipeline = append(pipeline, - bson.M{ - "$project": bson.M{ - "_id": 0, - "msg": "$msgs.msg", - }, - }, - bson.M{ - "$count": "count", - }, + + var ( + nextID primitive.ObjectID + count int + dataRange []searchMessageIndex + skip = int((req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber()) ) - count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) - if err != nil { - return 0, nil, err + _, _ = dataRange, skip + const maxDoc = 50 + data := make([]searchMessageIndex, 0, req.Pagination.GetShowNumber()) + push := cap(data) + for i := 0; ; i++ { + res, err := m.searchMessageIndex(ctx, filter, nextID, maxDoc) + if err != nil { + return 0, nil, err + } + if len(res) > 0 { + nextID = res[len(res)-1].ID + } + for _, r := range res { + var dataIndex []int64 + for _, index := range r.Index { + if push > 0 && count >= skip { + dataIndex = append(dataIndex, index) + push-- + } + count++ + } + if len(dataIndex) > 0 { + data = append(data, searchMessageIndex{ + ID: r.ID, + Index: dataIndex, + }) + } + } + if push <= 0 { + push-- + } + if len(res) < maxDoc || push < -10 { + return int64(count), data, nil + } + } +} + +func (m *MsgMgo) getDocRange(ctx context.Context, id primitive.ObjectID, index []int64) ([]*model.MsgInfoModel, error) { + if len(index) == 0 { + return nil, nil } - if len(count) == 0 || count[0] == 0 { - return 0, nil, nil + + pipeline := bson.A{ + bson.M{"$match": bson.M{"_id": id}}, + bson.M{"$project": "$msgs"}, } - pipeline = pipeline[:len(pipeline)-1] - pipeline = append(pipeline, - bson.M{ - "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), - }, - bson.M{ - "$limit": req.Pagination.GetShowNumber(), - }, - ) msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + return msgs, nil +} + +func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) { + count, data, err := m.searchMessage(ctx, req) if err != nil { return 0, nil, err } - for i := range msgs { - msgInfo := msgs[i] - if msgInfo == nil || msgInfo.Msg == nil { - continue + var msgs []*model.MsgInfoModel + if len(data) > 0 { + var n int + for _, d := range data { + n += len(d.Index) } - if msgInfo.Revoke != nil { - revokeContent := sdkws.MessageRevokedContent{ - RevokerID: msgInfo.Revoke.UserID, - RevokerRole: msgInfo.Revoke.Role, - ClientMsgID: msgInfo.Msg.ClientMsgID, - RevokerNickname: msgInfo.Revoke.Nickname, - RevokeTime: msgInfo.Revoke.Time, - SourceMessageSendTime: msgInfo.Msg.SendTime, - SourceMessageSendID: msgInfo.Msg.SendID, - SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, - SessionType: msgInfo.Msg.SessionType, - Seq: msgInfo.Msg.Seq, - Ex: msgInfo.Msg.Ex, - } - data, err := jsonutil.JsonMarshal(&revokeContent) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") - } - elem := sdkws.NotificationElem{Detail: string(data)} - content, err := jsonutil.JsonMarshal(&elem) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal elem") + msgs = make([]*model.MsgInfoModel, 0, n) + } + for _, val := range data { + res, err := mongoutil.FindOne[*model.MsgDocModel](ctx, m.coll, bson.M{"_id": val.ID}) + if err != nil { + return 0, nil, err + } + for _, i := range val.Index { + if i >= int64(len(res.Msg)) { + continue } - msgInfo.Msg.ContentType = constant.MsgRevokeNotification - msgInfo.Msg.Content = string(content) + msgs = append(msgs, res.Msg[i]) } } - //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber - //n := int32(len(msgs)) - //if start >= n { - // return n, []*relation.MsgInfoModel{}, nil - //} - //if start+req.Pagination.ShowNumber < n { - // msgs = msgs[start : start+req.Pagination.ShowNumber] - //} else { - // msgs = msgs[start:] - //} - return count[0], msgs, nil + return count, msgs, nil } +//func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { +// where := make(bson.A, 0, 6) +// if req.RecvID != "" { +// if req.SessionType == constant.ReadGroupChatType { +// where = append(where, bson.M{ +// "$or": bson.A{ +// bson.M{"doc_id": "^n_" + req.RecvID + ":"}, +// bson.M{"doc_id": "^sg_" + req.RecvID + ":"}, +// }, +// }) +// } else { +// where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) +// } +// } +// if req.SendID != "" { +// where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) +// } +// if req.ContentType != 0 { +// where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) +// } +// if req.SessionType != 0 { +// where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) +// } +// if req.SendTime != "" { +// sendTime, err := time.Parse(time.DateOnly, req.SendTime) +// if err != nil { +// return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) +// } +// where = append(where, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$gte": sendTime.UnixMilli(), +// }, +// }, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), +// }, +// }, +// ) +// } +// opt := options.Find().SetLimit(100) +// res, err := mongoutil.Find[model.MsgDocModel](ctx, m.coll, bson.M{"$and": where}, opt) +// if err != nil { +// return 0, nil, err +// } +// _ = res +// fmt.Println() +// +// return 0, nil, nil +// pipeline := bson.A{ +// bson.M{ +// "$unwind": "$msgs", +// }, +// } +// if len(where) > 0 { +// pipeline = append(pipeline, bson.M{ +// "$match": bson.M{"$and": where}, +// }) +// } +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 0, +// "msg": "$msgs.msg", +// }, +// }, +// bson.M{ +// "$count": "count", +// }, +// ) +// //count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) +// //if err != nil { +// // return 0, nil, err +// //} +// //if len(count) == 0 || count[0] == 0 { +// // return 0, nil, nil +// //} +// count := []int32{0} +// pipeline = pipeline[:len(pipeline)-1] +// pipeline = append(pipeline, +// bson.M{ +// "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), +// }, +// bson.M{ +// "$limit": req.Pagination.GetShowNumber(), +// }, +// ) +// msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, err +// } +// for i := range msgs { +// msgInfo := msgs[i] +// if msgInfo == nil || msgInfo.Msg == nil { +// continue +// } +// if msgInfo.Revoke != nil { +// revokeContent := sdkws.MessageRevokedContent{ +// RevokerID: msgInfo.Revoke.UserID, +// RevokerRole: msgInfo.Revoke.Role, +// ClientMsgID: msgInfo.Msg.ClientMsgID, +// RevokerNickname: msgInfo.Revoke.Nickname, +// RevokeTime: msgInfo.Revoke.Time, +// SourceMessageSendTime: msgInfo.Msg.SendTime, +// SourceMessageSendID: msgInfo.Msg.SendID, +// SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, +// SessionType: msgInfo.Msg.SessionType, +// Seq: msgInfo.Msg.Seq, +// Ex: msgInfo.Msg.Ex, +// } +// data, err := jsonutil.JsonMarshal(&revokeContent) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") +// } +// elem := sdkws.NotificationElem{Detail: string(data)} +// content, err := jsonutil.JsonMarshal(&elem) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal elem") +// } +// msgInfo.Msg.ContentType = constant.MsgRevokeNotification +// msgInfo.Msg.Content = string(content) +// } +// } +// //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber +// //n := int32(len(msgs)) +// //if start >= n { +// // return n, []*relation.MsgInfoModel{}, nil +// //} +// //if start+req.Pagination.ShowNumber < n { +// // msgs = msgs[start : start+req.Pagination.ShowNumber] +// //} else { +// // msgs = msgs[start:] +// //} +// return count[0], msgs, nil +//} + func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { var sort int if ase { diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go new file mode 100644 index 0000000000..5aed4dc511 --- /dev/null +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -0,0 +1,75 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "math/rand" + "strconv" + "testing" + "time" +) + +func TestName1(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + + req := &msg.SearchMessageReq{ + //RecvID: "3187706596", + //SendID: "7009965934", + ContentType: 101, + //SendTime: "2024-05-06", + //SessionType: 3, + Pagination: &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 10, + }, + } + total, res, err := v.SearchMessage(ctx, req) + if err != nil { + panic(err) + } + + for i, re := range res { + t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + } + + t.Log(total) +} + +func TestName10(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + opt := options.Find().SetLimit(1000) + + res, err := mongoutil.Find[model.MsgDocModel](ctx, v.coll, bson.M{}, opt) + if err != nil { + panic(err) + } + ctx = context.Background() + for i := 0; i < 100000; i++ { + for j := range res { + res[j].DocID = strconv.FormatUint(rand.Uint64(), 10) + ":0" + } + if err := mongoutil.InsertMany(ctx, v.coll, res); err != nil { + panic(err) + } + t.Log("====>", time.Now(), i) + } + +} diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index 5167314da9..42507a6937 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -26,7 +26,7 @@ func Mongodb() *mongo.Database { func TestUserSeq(t *testing.T) { uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) - t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4)) + t.Log(uSeq.SetUserMinSeq(context.Background(), "1000", "2000", 4)) } func TestConversationSeq(t *testing.T) { @@ -35,3 +35,8 @@ func TestConversationSeq(t *testing.T) { t.Log(cSeq.Malloc(context.Background(), "2000", 10)) t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) } + +func TestUserGetUserReadSeqs(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"})) +} diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 7c9a8f1336..9faad416ae 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -88,6 +88,14 @@ func (s *seqUserMongo) GetUserReadSeq(ctx context.Context, conversationID string return s.getSeq(ctx, conversationID, userID, "read_seq") } +func (s *seqUserMongo) notFoundSet0(seq map[string]int64, conversationIDs []string) { + for _, conversationID := range conversationIDs { + if _, ok := seq[conversationID]; !ok { + seq[conversationID] = 0 + } + } +} + func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { if len(conversationID) == 0 { return map[string]int64{}, nil @@ -102,6 +110,7 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve for _, seq := range seqs { res[seq.ConversationID] = seq.ReadSeq } + s.notFoundSet0(res, conversationID) return res, nil } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b402f3ac7d..84f3a9e3e2 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -37,7 +37,7 @@ type Msg interface { GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error - SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) + SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) From 882d24b063f924a48e7134d059841aabe82794be Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 29 Jul 2024 16:28:30 +0800 Subject: [PATCH 144/147] fix: ImportFriends --- pkg/common/storage/controller/friend.go | 61 +++++++++++++++---------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index e402f5980b..636371198a 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -152,42 +152,55 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return f.tx.Transaction(ctx, func(ctx context.Context) error { cache := f.cache.CloneFriendCache() // user find friends - fs1, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) + myFriends, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - opUserID := mcontext.GetOperationID(ctx) - for _, v := range friendUserIDs { - fs1 = append(fs1, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: v, AddSource: addSource, OperatorUserID: opUserID}) - } - fs11 := datautil.DistinctAny(fs1, func(e *model.Friend) string { - return e.FriendUserID - }) - - err = f.friend.Create(ctx, fs11) + addOwners, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - fs2, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) - if err != nil { - return err + opUserID := mcontext.GetOperationID(ctx) + friends := make([]*model.Friend, 0, len(friendUserIDs)*2) + myFriendsSet := datautil.SliceSetAny(myFriends, func(friend *model.Friend) string { + return friend.FriendUserID + }) + addOwnersSet := datautil.SliceSetAny(addOwners, func(friend *model.Friend) string { + return friend.OwnerUserID + }) + newMyFriendIDs := make([]string, 0, len(friendUserIDs)) + newMyOwnerIDs := make([]string, 0, len(friendUserIDs)) + for _, userID := range friendUserIDs { + if ownerUserID == userID { + continue + } + if _, ok := myFriendsSet[userID]; !ok { + myFriendsSet[userID] = struct{}{} + newMyFriendIDs = append(newMyFriendIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: userID, AddSource: addSource, OperatorUserID: opUserID}) + } + if _, ok := addOwnersSet[userID]; !ok { + addOwnersSet[userID] = struct{}{} + newMyOwnerIDs = append(newMyOwnerIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: userID, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) + } } - var newFriendIDs []string - for _, v := range friendUserIDs { - fs2 = append(fs2, &model.Friend{OwnerUserID: v, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) - newFriendIDs = append(newFriendIDs, v) + if len(friends) == 0 { + return nil } - fs22 := datautil.DistinctAny(fs2, func(e *model.Friend) string { - return e.OwnerUserID - }) - err = f.friend.Create(ctx, fs22) + err = f.friend.Create(ctx, friends) if err != nil { return err } - newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...) + if len(newMyFriendIDs) > 0 { + cache = cache.DelFriendIDs(newMyFriendIDs...) + cache = cache.DelFriends(ownerUserID, newMyFriendIDs).DelMaxFriendVersion(newMyFriendIDs...) + } + if len(newMyOwnerIDs) > 0 { + cache = cache.DelFriendIDs(newMyOwnerIDs...) + cache = cache.DelOwner(ownerUserID, newMyOwnerIDs).DelMaxFriendVersion(newMyOwnerIDs...) + } return cache.ChainExecDel(ctx) - }) } From 23f3445c2255368b2451b3716bcf79a1bb7ba59b Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 29 Jul 2024 17:31:04 +0800 Subject: [PATCH 145/147] add online cache --- internal/push/push_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ed87b3929f..1da008945b 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -198,6 +198,7 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. offlineUserIDs = append(offlineUserIDs, userID) } } + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "pushToUserIDs", pushToUserIDs, "onlineUserIDs", onlineUserIDs, "offlineUserIDs", offlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { var err error From eeede2149967095996d59226aa2dd8e97c1fb031 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 31 Jul 2024 10:27:44 +0800 Subject: [PATCH 146/147] add some logs --- internal/push/push_handler.go | 2 +- pkg/common/storage/cache/redis/online.go | 6 +- pkg/rpccache/online.go | 76 +++++++++++++----------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 1da008945b..c3b27372b3 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -198,7 +198,7 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. offlineUserIDs = append(offlineUserIDs, userID) } } - log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "pushToUserIDs", pushToUserIDs, "onlineUserIDs", onlineUserIDs, "offlineUserIDs", offlineUserIDs) + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { var err error diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index dc6a5f775b..a012e1cd2d 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -5,6 +5,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "strconv" "time" @@ -82,8 +83,11 @@ func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, o argv = append(argv, platformID) } keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} - if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + status, err := s.rdb.Eval(ctx, script, keys, argv).Result() + if err != nil { + log.ZError(ctx, "redis SetUserOnline", err, "userID", userID, "online", online, "offline", offline) return err } + log.ZDebug(ctx, "redis SetUserOnline", "userID", userID, "online", online, "offline", offline, "status", status) return nil } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5db68d198d..372a7efe82 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -28,7 +28,7 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) if err != nil { - log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + log.ZError(ctx, "OnlineCache setUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } storageCache := x.setUserOnline(userID, platformIDs) @@ -48,9 +48,15 @@ type OnlineCache struct { } func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - return o.local.Get(userID, func() ([]int32, error) { + platformIDs, err := o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) + return nil, err + } + log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + return nil, err } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { @@ -61,39 +67,39 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } -func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { - onlineUserIDs := make([]string, 0, len(userIDs)) - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) - return onlineUserIDs, nil -} - -func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { - userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - var onlineUserIDs []string - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) - return onlineUserIDs, nil -} +//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { +// onlineUserIDs := make([]string, 0, len(userIDs)) +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) +// return onlineUserIDs, nil +//} +// +//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// var onlineUserIDs []string +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) +// return onlineUserIDs, nil +//} func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { return o.local.SetHas(userID, platformIDs) From 05a3810b05fe7beb62f1c19a937265793eed80e3 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 31 Jul 2024 12:32:31 +0800 Subject: [PATCH 147/147] add some logs --- pkg/rpccache/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 372a7efe82..13578a7df6 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -56,7 +56,7 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return nil, err } log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) - return nil, err + return platformIDs, nil } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) {