From 381fd1beebec9cfc252625ee8cdd2f5301c22029 Mon Sep 17 00:00:00 2001 From: coderwander <770732124@qq.com> Date: Tue, 16 Apr 2024 11:30:20 +0800 Subject: [PATCH 01/91] chore: fix some typos in comments Signed-off-by: coderwander <770732124@qq.com> --- config/templates/env.template | 2 +- deployments/templates/env-template.yaml | 2 +- pkg/common/db/controller/msg.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/templates/env.template b/config/templates/env.template index 7a095b2bbd..1441f8af9f 100644 --- a/config/templates/env.template +++ b/config/templates/env.template @@ -129,7 +129,7 @@ REDIS_PORT=16379 # Default: REDIS_PASSWORD=openIM123 REDIS_PASSWORD=openIM123 -# Kakfa username to authenticate with the Kafka service. +# Kafka username to authenticate with the Kafka service. # KAFKA_USERNAME='' # Port on which Kafka distributed streaming platform is running. diff --git a/deployments/templates/env-template.yaml b/deployments/templates/env-template.yaml index b1fb8b78be..b019f97e5b 100644 --- a/deployments/templates/env-template.yaml +++ b/deployments/templates/env-template.yaml @@ -129,7 +129,7 @@ REDIS_PORT=${REDIS_PORT} # Default: REDIS_PASSWORD=openIM123 REDIS_PASSWORD=${REDIS_PASSWORD} -# Kakfa username to authenticate with the Kafka service. +# Kafka username to authenticate with the Kafka service. # KAFKA_USERNAME=${KAFKA_USERNAME} # Port on which Kafka distributed streaming platform is running. diff --git a/pkg/common/db/controller/msg.go b/pkg/common/db/controller/msg.go index ccf209b7a1..722fc8f903 100644 --- a/pkg/common/db/controller/msg.go +++ b/pkg/common/db/controller/msg.go @@ -841,7 +841,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio } log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg)) if int64(len(msgDocModel.Msg)) > db.msg.GetSingleGocMsgNum() { - log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) + log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) } if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < utils.GetCurrentTimestampByMill() { log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID) From 6ab1244ad8544a2f506abf2b0eec96aaa346b210 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Thu, 23 May 2024 10:19:54 +0800 Subject: [PATCH 02/91] merge --- CHANGELOG/CHANGELOG-1.0.md | 56 ----------- CHANGELOG/CHANGELOG-2.0.md | 87 ----------------- CHANGELOG/CHANGELOG-2.1.md | 50 ---------- CHANGELOG/CHANGELOG-2.2.md | 50 ---------- CHANGELOG/CHANGELOG-2.3.md | 70 -------------- CHANGELOG/CHANGELOG-2.9.md | 52 ---------- CHANGELOG/CHANGELOG-3.0.md | 188 ------------------------------------- CHANGELOG/CHANGELOG-3.1.md | 20 ---- CHANGELOG/CHANGELOG-3.2.md | 32 ------- CHANGELOG/CHANGELOG-3.3.md | 40 -------- CHANGELOG/CHANGELOG-3.4.md | 32 ------- CHANGELOG/CHANGELOG-3.5.md | 84 ----------------- CHANGELOG/CHANGELOG-3.6.md | 20 ---- CHANGELOG/CHANGELOG.md | 173 ---------------------------------- 14 files changed, 954 deletions(-) delete mode 100644 CHANGELOG/CHANGELOG-1.0.md delete mode 100644 CHANGELOG/CHANGELOG-2.0.md delete mode 100644 CHANGELOG/CHANGELOG-2.1.md delete mode 100644 CHANGELOG/CHANGELOG-2.2.md delete mode 100644 CHANGELOG/CHANGELOG-2.3.md delete mode 100644 CHANGELOG/CHANGELOG-2.9.md delete mode 100644 CHANGELOG/CHANGELOG-3.0.md delete mode 100644 CHANGELOG/CHANGELOG-3.1.md delete mode 100644 CHANGELOG/CHANGELOG-3.2.md delete mode 100644 CHANGELOG/CHANGELOG-3.3.md delete mode 100644 CHANGELOG/CHANGELOG-3.4.md delete mode 100644 CHANGELOG/CHANGELOG-3.5.md delete mode 100644 CHANGELOG/CHANGELOG-3.6.md delete mode 100644 CHANGELOG/CHANGELOG.md diff --git a/CHANGELOG/CHANGELOG-1.0.md b/CHANGELOG/CHANGELOG-1.0.md deleted file mode 100644 index bed425943c..0000000000 --- a/CHANGELOG/CHANGELOG-1.0.md +++ /dev/null @@ -1,56 +0,0 @@ -# Version logging for OpenIM - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v1.0.7 - 2021-12-17](#v107---2021-12-17) - - [v1.0.6 - 2021-12-10](#v106---2021-12-10) - - [v1.0.5 - 2021-12-03](#v105---2021-12-03) - - [v1.0.4 - 2021-11-25](#v104---2021-11-25) - - [v1.0.3 - 2021-11-12](#v103---2021-11-12) - - [v1.0.1 - 2021-11-04](#v101---2021-11-04) - - [v1.0.0 - 2021-10-28](#v100---2021-10-28) - - [Reverts](#reverts) - - - - -## [Unreleased] - - - -## [v1.0.7] - 2021-12-17 - - -## [v1.0.6] - 2021-12-10 - - -## [v1.0.5] - 2021-12-03 - - -## [v1.0.4] - 2021-11-25 - - -## [v1.0.3] - 2021-11-12 - - -## [v1.0.1] - 2021-11-04 - - -## v1.0.0 - 2021-10-28 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v1.0.7...HEAD -[v1.0.7]: https://github.com/openimsdk/open-im-server/compare/v1.0.6...v1.0.7 -[v1.0.6]: https://github.com/openimsdk/open-im-server/compare/v1.0.5...v1.0.6 -[v1.0.5]: https://github.com/openimsdk/open-im-server/compare/v1.0.4...v1.0.5 -[v1.0.4]: https://github.com/openimsdk/open-im-server/compare/v1.0.3...v1.0.4 -[v1.0.3]: https://github.com/openimsdk/open-im-server/compare/v1.0.1...v1.0.3 -[v1.0.1]: https://github.com/openimsdk/open-im-server/compare/v1.0.0...v1.0.1 diff --git a/CHANGELOG/CHANGELOG-2.0.md b/CHANGELOG/CHANGELOG-2.0.md deleted file mode 100644 index 8902f18147..0000000000 --- a/CHANGELOG/CHANGELOG-2.0.md +++ /dev/null @@ -1,87 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v2.0.10] - 2022-05-13 - - -## [v2.0.9] - 2022-04-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.8] - 2022-04-24 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.7] - 2022-04-08 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.6] - 2022-04-01 -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.0.5] - 2022-03-24 - - -## [v2.04] - 2022-03-18 - - -## [v2.0.3] - 2022-03-11 - - -## [v2.0.2] - 2022-03-04 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.1] - 2022-02-25 - - -## v2.0.0 - 2022-02-23 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.0.10...HEAD -[v2.0.10]: https://github.com/openimsdk/open-im-server/compare/v2.0.9...v2.0.10 -[v2.0.9]: https://github.com/openimsdk/open-im-server/compare/v2.0.8...v2.0.9 -[v2.0.8]: https://github.com/openimsdk/open-im-server/compare/v2.0.7...v2.0.8 -[v2.0.7]: https://github.com/openimsdk/open-im-server/compare/v2.0.6...v2.0.7 -[v2.0.6]: https://github.com/openimsdk/open-im-server/compare/v2.0.5...v2.0.6 -[v2.0.5]: https://github.com/openimsdk/open-im-server/compare/v2.04...v2.0.5 -[v2.04]: https://github.com/openimsdk/open-im-server/compare/v2.0.3...v2.04 -[v2.0.3]: https://github.com/openimsdk/open-im-server/compare/v2.0.2...v2.0.3 -[v2.0.2]: https://github.com/openimsdk/open-im-server/compare/v2.0.1...v2.0.2 -[v2.0.1]: https://github.com/openimsdk/open-im-server/compare/v2.0.0...v2.0.1 diff --git a/CHANGELOG/CHANGELOG-2.1.md b/CHANGELOG/CHANGELOG-2.1.md deleted file mode 100644 index 7eef06f08a..0000000000 --- a/CHANGELOG/CHANGELOG-2.1.md +++ /dev/null @@ -1,50 +0,0 @@ -# Version logging for OpenIM:v2.1 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.1](#version-logging-for-openimv21) - - [Unreleased](#unreleased) - - [v2.1.0 - 2022-06-17](#v210---2022-06-17) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## v2.1.0 - 2022-06-17 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.1.0...HEAD diff --git a/CHANGELOG/CHANGELOG-2.2.md b/CHANGELOG/CHANGELOG-2.2.md deleted file mode 100644 index 8afb24b29f..0000000000 --- a/CHANGELOG/CHANGELOG-2.2.md +++ /dev/null @@ -1,50 +0,0 @@ -# Version logging for OpenIM:v2.2 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.2](#version-logging-for-openimv22) - - [Unreleased](#unreleased) - - [v2.2.0 - 2022-07-01](#v220---2022-07-01) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## v2.2.0 - 2022-07-01 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.2.0...HEAD diff --git a/CHANGELOG/CHANGELOG-2.3.md b/CHANGELOG/CHANGELOG-2.3.md deleted file mode 100644 index e14745f30d..0000000000 --- a/CHANGELOG/CHANGELOG-2.3.md +++ /dev/null @@ -1,70 +0,0 @@ -# Version logging for OpenIM:v2.3 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.3](#version-logging-for-openimv23) - - [Unreleased](#unreleased) - - [v2.3.3 - 2022-09-18](#v233---2022-09-18) - - [v2.3.2 - 2022-09-09](#v232---2022-09-09) - - [v2.3.0-rc2 - 2022-07-29](#v230-rc2---2022-07-29) - - [v2.3.0-rc1 - 2022-07-25](#v230-rc1---2022-07-25) - - [v2.3.0-rc0 - 2022-07-15](#v230-rc0---2022-07-15) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## [v2.3.3] - 2022-09-18 - - -## [v2.3.2] - 2022-09-09 - - -## [v2.3.0-rc2] - 2022-07-29 - - -## [v2.3.0-rc1] - 2022-07-25 - - -## v2.3.0-rc0 - 2022-07-15 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.3.3...HEAD -[v2.3.3]: https://github.com/openimsdk/open-im-server/compare/v2.3.2...v2.3.3 -[v2.3.2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc2...v2.3.2 -[v2.3.0-rc2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc1...v2.3.0-rc2 -[v2.3.0-rc1]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc0...v2.3.0-rc1 diff --git a/CHANGELOG/CHANGELOG-2.9.md b/CHANGELOG/CHANGELOG-2.9.md deleted file mode 100644 index b921eb51b8..0000000000 --- a/CHANGELOG/CHANGELOG-2.9.md +++ /dev/null @@ -1,52 +0,0 @@ -# Version logging for OpenIM - - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v2.9.0+1.839643f - 2023-07-07](#v2901839643f---2023-07-07) - - [v2.9.0+2.35f07fe - 2023-07-06](#v290235f07fe---2023-07-06) - - [v2.9.0+1.b5072b1 - 2023-07-05](#v2901b5072b1---2023-07-05) - - [v2.9.0+3.2667a3a - 2023-07-05](#v29032667a3a---2023-07-05) - - [v2.9.0+7.04818ca - 2023-07-05](#v290704818ca---2023-07-05) - - [v2.9.0 - 2023-07-04](#v290---2023-07-04) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## [v2.9.0+1.839643f] - 2023-07-07 - - -## [v2.9.0+2.35f07fe] - 2023-07-06 - - -## [v2.9.0+1.b5072b1] - 2023-07-05 - - -## [v2.9.0+3.2667a3a] - 2023-07-05 - - -## [v2.9.0+7.04818ca] - 2023-07-05 - - -## v2.9.0 - 2023-07-04 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+1.839643f...HEAD -[v2.9.0+1.839643f]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+2.35f07fe...v2.9.0+1.839643f -[v2.9.0+2.35f07fe]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+1.b5072b1...v2.9.0+2.35f07fe -[v2.9.0+1.b5072b1]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+3.2667a3a...v2.9.0+1.b5072b1 -[v2.9.0+3.2667a3a]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+7.04818ca...v2.9.0+3.2667a3a -[v2.9.0+7.04818ca]: https://github.com/openimsdk/open-im-server/compare/v2.9.0...v2.9.0+7.04818ca diff --git a/CHANGELOG/CHANGELOG-3.0.md b/CHANGELOG/CHANGELOG-3.0.md deleted file mode 100644 index 1b99e7e34a..0000000000 --- a/CHANGELOG/CHANGELOG-3.0.md +++ /dev/null @@ -1,188 +0,0 @@ -# Version logging for OpenIM - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v2.9.0 - 2023-07-04](#v290---2023-07-04) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - [v2.3.3 - 2022-09-18](#v233---2022-09-18) - - [v2.3.2 - 2022-09-09](#v232---2022-09-09) - - [v2.3.0-rc2 - 2022-07-29](#v230-rc2---2022-07-29) - - [v2.3.0-rc1 - 2022-07-25](#v230-rc1---2022-07-25) - - [v2.3.0-rc0 - 2022-07-15](#v230-rc0---2022-07-15) - - [v2.2.0 - 2022-07-01](#v220---2022-07-01) - - [v2.1.0 - 2022-06-17](#v210---2022-06-17) - - [Pull Requests](#pull-requests-1) - - [v2.0.10 - 2022-05-13](#v2010---2022-05-13) - - [v2.0.9 - 2022-04-29](#v209---2022-04-29) - - [Reverts](#reverts-1) - - [Pull Requests](#pull-requests-2) - - [v2.0.7 - 2022-04-08](#v207---2022-04-08) - - [Pull Requests](#pull-requests-3) - - [v2.0.6 - 2022-04-01](#v206---2022-04-01) - - [Pull Requests](#pull-requests-4) - - [v2.0.5 - 2022-03-24](#v205---2022-03-24) - - [v2.04 - 2022-03-18](#v204---2022-03-18) - - [v2.0.3 - 2022-03-11](#v203---2022-03-11) - - [v2.0.2 - 2022-03-04](#v202---2022-03-04) - - [Pull Requests](#pull-requests-5) - - [v2.0.1 - 2022-02-25](#v201---2022-02-25) - - [v2.0.0 - 2022-02-23](#v200---2022-02-23) - - [v1.0.7 - 2021-12-17](#v107---2021-12-17) - - [v1.0.6 - 2021-12-10](#v106---2021-12-10) - - [v1.0.5 - 2021-12-03](#v105---2021-12-03) - - [v1.0.4 - 2021-11-25](#v104---2021-11-25) - - [v1.0.3 - 2021-11-12](#v103---2021-11-12) - - [v1.0.1 - 2021-11-04](#v101---2021-11-04) - - [v1.0.0 - 2021-10-28](#v100---2021-10-28) - - [Reverts](#reverts-2) - - - - -## [Unreleased] - - - -## [v2.9.0] - 2023-07-04 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.3.3] - 2022-09-18 - - -## [v2.3.2] - 2022-09-09 - - -## [v2.3.0-rc2] - 2022-07-29 - - -## [v2.3.0-rc1] - 2022-07-25 - - -## [v2.3.0-rc0] - 2022-07-15 - - -## [v2.2.0] - 2022-07-01 - - -## [v2.1.0] - 2022-06-17 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.10] - 2022-05-13 - - -## [v2.0.9] - 2022-04-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.7] - 2022-04-08 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.6] - 2022-04-01 -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.0.5] - 2022-03-24 - - -## [v2.04] - 2022-03-18 - - -## [v2.0.3] - 2022-03-11 - - -## [v2.0.2] - 2022-03-04 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.1] - 2022-02-25 - - -## [v2.0.0] - 2022-02-23 - - -## [v1.0.7] - 2021-12-17 - - -## [v1.0.6] - 2021-12-10 - - -## [v1.0.5] - 2021-12-03 - - -## [v1.0.4] - 2021-11-25 - - -## [v1.0.3] - 2021-11-12 - - -## [v1.0.1] - 2021-11-04 - - -## v1.0.0 - 2021-10-28 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.9.0...HEAD -[v2.9.0]: https://github.com/openimsdk/open-im-server/compare/v2.3.3...v2.9.0 -[v2.3.3]: https://github.com/openimsdk/open-im-server/compare/v2.3.2...v2.3.3 -[v2.3.2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc2...v2.3.2 -[v2.3.0-rc2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc1...v2.3.0-rc2 -[v2.3.0-rc1]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc0...v2.3.0-rc1 -[v2.3.0-rc0]: https://github.com/openimsdk/open-im-server/compare/v2.2.0...v2.3.0-rc0 -[v2.2.0]: https://github.com/openimsdk/open-im-server/compare/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/openimsdk/open-im-server/compare/v2.0.10...v2.1.0 -[v2.0.10]: https://github.com/openimsdk/open-im-server/compare/v2.0.9...v2.0.10 -[v2.0.9]: https://github.com/openimsdk/open-im-server/compare/v2.0.7...v2.0.9 -[v2.0.7]: https://github.com/openimsdk/open-im-server/compare/v2.0.6...v2.0.7 -[v2.0.6]: https://github.com/openimsdk/open-im-server/compare/v2.0.5...v2.0.6 -[v2.0.5]: https://github.com/openimsdk/open-im-server/compare/v2.04...v2.0.5 -[v2.04]: https://github.com/openimsdk/open-im-server/compare/v2.0.3...v2.04 -[v2.0.3]: https://github.com/openimsdk/open-im-server/compare/v2.0.2...v2.0.3 -[v2.0.2]: https://github.com/openimsdk/open-im-server/compare/v2.0.1...v2.0.2 -[v2.0.1]: https://github.com/openimsdk/open-im-server/compare/v2.0.0...v2.0.1 -[v2.0.0]: https://github.com/openimsdk/open-im-server/compare/v1.0.7...v2.0.0 -[v1.0.7]: https://github.com/openimsdk/open-im-server/compare/v1.0.6...v1.0.7 -[v1.0.6]: https://github.com/openimsdk/open-im-server/compare/v1.0.5...v1.0.6 -[v1.0.5]: https://github.com/openimsdk/open-im-server/compare/v1.0.4...v1.0.5 -[v1.0.4]: https://github.com/openimsdk/open-im-server/compare/v1.0.3...v1.0.4 -[v1.0.3]: https://github.com/openimsdk/open-im-server/compare/v1.0.1...v1.0.3 -[v1.0.1]: https://github.com/openimsdk/open-im-server/compare/v1.0.0...v1.0.1 diff --git a/CHANGELOG/CHANGELOG-3.1.md b/CHANGELOG/CHANGELOG-3.1.md deleted file mode 100644 index 8944321830..0000000000 --- a/CHANGELOG/CHANGELOG-3.1.md +++ /dev/null @@ -1,20 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## v3.1.0 - 2023-07-28 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.1.0...HEAD diff --git a/CHANGELOG/CHANGELOG-3.2.md b/CHANGELOG/CHANGELOG-3.2.md deleted file mode 100644 index 3f77a82738..0000000000 --- a/CHANGELOG/CHANGELOG-3.2.md +++ /dev/null @@ -1,32 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.2.2-alpha.0] - 2023-08-25 - - -## [v3.2.0] - 2023-08-19 - - -## [v3.2.0-rc.0] - 2023-08-17 - - -## v3.2.0-alpha.0 - 2023-08-16 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-alpha.0...HEAD -[v3.2.2-alpha.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0...v3.2.2-alpha.0 -[v3.2.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-rc.0...v3.2.0 -[v3.2.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-alpha.0...v3.2.0-rc.0 diff --git a/CHANGELOG/CHANGELOG-3.3.md b/CHANGELOG/CHANGELOG-3.3.md deleted file mode 100644 index f7326691ac..0000000000 --- a/CHANGELOG/CHANGELOG-3.3.md +++ /dev/null @@ -1,40 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.3.1] - 2023-09-13 - - -## [v3.3.1-beta.0] - 2023-09-11 - - -## [v3.3.0-rc.1] - 2023-09-11 - - -## [v3.3.0-rc.12] - 2023-09-11 - - -## [v3.3.0] - 2023-09-09 - - -## v3.3.0-rc.0 - 2023-09-07 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.3.1...HEAD -[v3.3.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.1-beta.0...v3.3.1 -[v3.3.1-beta.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.1...v3.3.1-beta.0 -[v3.3.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.12...v3.3.0-rc.1 -[v3.3.0-rc.12]: https://github.com/openimsdk/open-im-server/compare/v3.3.0...v3.3.0-rc.12 -[v3.3.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.0...v3.3.0 diff --git a/CHANGELOG/CHANGELOG-3.4.md b/CHANGELOG/CHANGELOG-3.4.md deleted file mode 100644 index db4dd59c53..0000000000 --- a/CHANGELOG/CHANGELOG-3.4.md +++ /dev/null @@ -1,32 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.4.2] - 2023-12-14 - - -## [v3.4.0] - 2023-11-10 - - -## [v3.4.0-rc.1] - 2023-11-09 - - -## v3.4.0-rc.0 - 2023-11-09 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.4.2...HEAD -[v3.4.2]: https://github.com/openimsdk/open-im-server/compare/v3.4.0...v3.4.2 -[v3.4.0]: https://github.com/openimsdk/open-im-server/compare/v3.4.0-rc.1...v3.4.0 -[v3.4.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.4.0-rc.0...v3.4.0-rc.1 diff --git a/CHANGELOG/CHANGELOG-3.5.md b/CHANGELOG/CHANGELOG-3.5.md deleted file mode 100644 index b929922f16..0000000000 --- a/CHANGELOG/CHANGELOG-3.5.md +++ /dev/null @@ -1,84 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.5.1-alpha.2] - 2024-01-26 - - -## [v3.5.1-rc.1] - 2024-01-23 - - -## [v3.5.1-alpha.1] - 2024-01-09 - - -## [v3.5.0] - 2024-01-02 - - -## [v3.5.1] - 2024-01-02 - - -## [v3.5.1-bate.1] - 2024-01-02 - - -## [v3.5.1-rc.0] - 2023-12-30 - - -## [v3.5.0-rc.8] - 2023-12-28 - - -## [v3.5.0-rc.7] - 2023-12-18 - - -## [v3.5.0-rc.6] - 2023-12-15 - - -## [v3.5.0-rc.5] - 2023-12-15 - - -## [v3.5.0-rc.4] - 2023-12-14 - - -## [v3.5.0-rc.3] - 2023-12-14 - - -## [v3.5.0-rc.2] - 2023-12-14 - - -## [v3.5.0-rc.1] - 2023-12-14 - - -## [v3.5.0-rc.0] - 2023-12-14 - - -## v3.5.0-beta.1 - 2023-11-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-alpha.2...HEAD -[v3.5.1-alpha.2]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-rc.1...v3.5.1-alpha.2 -[v3.5.1-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-alpha.1...v3.5.1-rc.1 -[v3.5.1-alpha.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.0...v3.5.1-alpha.1 -[v3.5.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.1...v3.5.0 -[v3.5.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-bate.1...v3.5.1 -[v3.5.1-bate.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-rc.0...v3.5.1-bate.1 -[v3.5.1-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.8...v3.5.1-rc.0 -[v3.5.0-rc.8]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.7...v3.5.0-rc.8 -[v3.5.0-rc.7]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.6...v3.5.0-rc.7 -[v3.5.0-rc.6]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.5...v3.5.0-rc.6 -[v3.5.0-rc.5]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.4...v3.5.0-rc.5 -[v3.5.0-rc.4]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.3...v3.5.0-rc.4 -[v3.5.0-rc.3]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.2...v3.5.0-rc.3 -[v3.5.0-rc.2]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.1...v3.5.0-rc.2 -[v3.5.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.0...v3.5.0-rc.1 -[v3.5.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-beta.1...v3.5.0-rc.0 diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md deleted file mode 100644 index 214d583401..0000000000 --- a/CHANGELOG/CHANGELOG-3.6.md +++ /dev/null @@ -1,20 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## v3.6.0 - 2024-03-07 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.6.0...HEAD diff --git a/CHANGELOG/CHANGELOG.md b/CHANGELOG/CHANGELOG.md deleted file mode 100644 index f5d62ab242..0000000000 --- a/CHANGELOG/CHANGELOG.md +++ /dev/null @@ -1,173 +0,0 @@ -# Changelog - -- [Changelog](#changelog) - - [OpenIM versioning policy](#openim-versioning-policy) - - [command](#command) - - [install](#install) - - [User](#user) - - [create next tag](#create-next-tag) - - [Release version logs](#release-version-logs) - - [Introduction](#introduction) - - [Naming Format](#naming-format) - - [Examples](#examples) - - [Version Modifiers](#version-modifiers) - - [Versioning Strategy](#versioning-strategy) - - -All notable changes to this project will be documented in this file. - -+ [https://github.com/openimsdk/open-im-server/releases](https://github.com/openimsdk/open-im-server/releases) - -## OpenIM versioning policy - -+ [OpenIM Version](../docs/contrib/version.md) - -## command - -To use git-chglog you need to configure: - -1. CHANGELOG templates -2. git-chglog configuration - -### install - -```bash -$ go get github.com/git-chglog/git-chglog/cmd/git-chglog -``` - - -## User - -```bash -$ git-chglog --init -``` - -**Options** - -- What is the URL of your repository?: https://github.com/openimsdk/open-im-server -- What is your favorite style?: github -- Choose the format of your favorite commit message: (): -- feat(core): Add new feature -- What is your favorite template style?: standard -- Do you include Merge Commit in CHANGELOG?: n -- Do you include Revert Commit in CHANGELOG?: y -- In which directory do you output configuration files and templates?: .chglog - -```bash -git-chglog --tag-filter-pattern 'v2.0.*' -o CHANGELOG-2.0.md -``` - -**Other uses:** - -```bash -$ git-chglog - - If is not specified, it corresponds to all tags. - This is the simplest example. - -$ git-chglog 1.0.0..2.0.0 - - The above is a command to generate CHANGELOG including commit of 1.0.0 to 2.0.0. - -$ git-chglog 1.0.0 - - The above is a command to generate CHANGELOG including commit of only 1.0.0. - -$ git-chglog $(git describe --tags $(git rev-list --tags --max-count=1)) - - The above is a command to generate CHANGELOG with the commit included in the latest tag. - -$ git-chglog --output CHANGELOG.md - - The above is a command to output to CHANGELOG.md instead of standard output. - -$ git-chglog --config custom/dir/config.yml - - The above is a command that uses a configuration file placed other than ".chglog/config.yml". -``` - - -## create next tag - -```bash -$ git-chglog --next-tag 2.0.0 -o CHANGELOG.md -$ git commit -am "release 2.0.0" -$ git tag 2.0.0 -``` - -| Query | Description | Example | -| -------------- | ---------------------------------------------- | --------------------------- | -| `..` | Commit contained in `` tags from ``. | `$ git-chglog 1.0.0..2.0.0` | -| `..` | Commit from the `` to the latest tag. | `$ git-chglog 1.0.0..` | -| `..` | Commit from the oldest tag to ``. | `$ git-chglog ..2.0.0` | -| `` | Commit contained in ``. | `$ git-chglog 1.0.0` | - - -## Release version logs - -+ [OpenIM CHANGELOG-V1.0](CHANGELOG-1.0.md) -+ [OpenIM CHANGELOG-V2.0](CHANGELOG-2.0.md) -+ [OpenIM CHANGELOG-V2.1](CHANGELOG-2.1.md) -+ [OpenIM CHANGELOG-V2.2](CHANGELOG-2.2.md) -+ [OpenIM CHANGELOG-V2.3](CHANGELOG-2.3.md) -+ [OpenIM CHANGELOG-V2.9](CHANGELOG-2.9.md) -+ [OpenIM CHANGELOG-V3.0](CHANGELOG-3.0.md) -+ [OpenIM CHANGELOG-V3.1](CHANGELOG-3.1.md) -+ [OpenIM CHANGELOG-V3.2](CHANGELOG-3.2.md) -+ [OpenIM CHANGELOG-V3.3](CHANGELOG-3.3.md) - - -## Introduction - -In both the open-source and closed-source software development communities, it is important to follow a consistent and understandable versioning scheme for software projects. This ensures clear communication of changes, compatibility, and stability across different releases. One widely adopted naming convention is the Semantic Versioning 2.0.0. - -## Naming Format - -The most common format for version numbers is as follows: - -```bash -major.minor[.patch[.build]] -``` - -Let's take a closer look at each component: - -1. **Major Version**: This is the first number in the versioning scheme and indicates significant changes that may not be backward compatible (specific to each project). -2. **Minor Version**: The second number signifies the addition of new features while maintaining backward compatibility. -3. **Patch Version**: The third number represents bug fixes or code optimizations without introducing new features. It is generally backward compatible. -4. **Build Version**: Typically an automatically generated number that increments with each code commit. - -## Examples - -Here are a few examples to illustrate the versioning scheme: - -1. `1.0` -2. `2.14.0.1478` -3. `3.2.1 build-354` - -## Version Modifiers - -Apart from the version numbers, there are also version modifiers used to indicate specific stages or statuses of a release. Some commonly used version modifiers include: - -- **alpha**: An internal testing version with numerous known bugs. It is primarily used for communication among developers. -- **beta**: A testing version released to enthusiastic users for feedback and bug detection. -- **rc (release candidate)**: The final testing version before the official release. -- **ga (general availability)**: The initial stable release for public distribution. -- **r/release** (or no modifier at all): The final released version intended for general users. -- **lts (long-term support)**: Designates a version that will receive extended maintenance and bug fixes for a specified number of years. - -## Versioning Strategy - -To effectively manage version numbers, the following strategies are commonly employed: - -- The initial version of a project can be either `0.1` or `1.0`. -- When fixing bugs, the patch version is incremented by 1. -- When adding new features, the minor version is incremented by 1, and the patch version is reset to 0. -- In the case of significant modifications, the major version is incremented by 1. -- The build version is usually automatically generated by the compilation process and follows a defined format. It does not require manual control. - -By adhering to these strategies and guidelines, developers can maintain consistency and clarity in versioning their software projects. This enables users and collaborators to understand the nature of changes between different releases and ensure compatibility with their systems. - -(Note: Markdown formatting has been used to structure this article. Markdown is a lightweight markup language used to format text on platforms like GitHub.) - ------- - -**Note**: The above article is based on the given content and aims to provide a Markdown-formatted English article explaining the naming conventions for software project versions, specifically focusing on the Semantic Versioning 2.0.0. \ No newline at end of file From e6f1232582b0e7e01f5ca5594ae9d4c2c2948611 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:54:42 +0800 Subject: [PATCH 03/91] fix: message can store Ex (#2371) --- internal/api/msg.go | 1 + pkg/apistruct/manage.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/internal/api/msg.go b/internal/api/msg.go index 180342e591..ba63fbb66f 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -101,6 +101,7 @@ func (m MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) SendTime: params.SendTime, Options: options, OfflinePushInfo: params.OfflinePushInfo, + Ex: params.Ex, }, } return &pbData diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index e79b477222..6ea6a29ed6 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -55,6 +55,9 @@ type SendMsg struct { // OfflinePushInfo contains information for offline push notifications. OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + + // Ex stores extended fields + Ex string `json:"ex"` } // SendMsgReq extends SendMsg with the requirement of RecvID when SessionType indicates a one-on-one or notification chat. From 118c5f56f3e671d28248ff3d90529eb9cb1d5376 Mon Sep 17 00:00:00 2001 From: printlin <32053356+printlin@users.noreply.github.com> Date: Thu, 27 Jun 2024 18:10:06 +0800 Subject: [PATCH 04/91] feature: add webhook AttentionIds (#2370) --- config/webhooks.yml | 3 +++ internal/rpc/msg/callback.go | 5 +++++ pkg/common/config/config.go | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/webhooks.yml b/config/webhooks.yml index c7839d4f21..11a85ba0c4 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -13,6 +13,9 @@ afterUpdateUserInfoEx: afterSendSingleMsg: enable: false timeout: 5 + # Only the senID/recvID specified in attentionIds will send the callback + # if not set, all user messages will be callback + attentionIds: [] beforeSendGroupMsg: enable: false timeout: 5 diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index 10404675eb..be58d75047 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -83,6 +83,11 @@ func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config if msg.MsgData.ContentType == constant.Typing { return } + // According to the attentionIds configuration, only some users are sent + attentionIds := after.AttentionIds + if attentionIds != nil && !datautil.Contain(msg.MsgData.RecvID, attentionIds...) && !datautil.Contain(msg.MsgData.SendID, attentionIds...) { + return + } cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), RecvID: msg.MsgData.RecvID, diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f5..0b6176fb74 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -339,8 +339,9 @@ type BeforeConfig struct { } type AfterConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` + Enable bool `mapstructure:"enable"` + Timeout int `mapstructure:"timeout"` + AttentionIds []string `mapstructure:"attentionIds"` } type Share struct { From fe7c029c2a7562fac400279cb0b41b8110cbf818 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 27 Jun 2024 18:16:03 +0800 Subject: [PATCH 05/91] fix: group application (#2367) * fix: group application * feat: constant --- internal/rpc/group/notification.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index cfa62c85db..a9abb03e6a 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -17,13 +17,13 @@ package group import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" @@ -34,6 +34,12 @@ import ( "github.com/openimsdk/tools/utils/stringutil" ) +// GroupApplicationReceiver +const ( + applicantReceiver = iota + adminReceiver +) + func NewGroupNotificationSender(db controller.GroupDatabase, msgRpcClient *rpcclient.MessageRpcClient, userRpcClient *rpcclient.UserRpcClient, config *Config, fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error)) *GroupNotificationSender { return &GroupNotificationSender{ NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(msgRpcClient), rpcclient.WithUserRpcClient(userRpcClient)), @@ -400,15 +406,17 @@ func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx conte if err != nil { return } - tips := &sdkws.GroupApplicationAcceptedTips{Group: group, HandleMsg: req.HandledMsg} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + + var opUser *sdkws.GroupMemberFullInfo + if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } for _, userID := range append(userIDs, req.FromUserID) { + tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { - tips.ReceiverAs = 0 + tips.ReceiverAs = applicantReceiver } else { - tips.ReceiverAs = 1 + tips.ReceiverAs = adminReceiver } g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationAcceptedNotification, tips) } @@ -431,15 +439,17 @@ func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx conte if err != nil { return } - tips := &sdkws.GroupApplicationRejectedTips{Group: group, HandleMsg: req.HandledMsg} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + + var opUser *sdkws.GroupMemberFullInfo + if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } for _, userID := range append(userIDs, req.FromUserID) { + tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { - tips.ReceiverAs = 0 + tips.ReceiverAs = applicantReceiver } else { - tips.ReceiverAs = 1 + tips.ReceiverAs = adminReceiver } g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationRejectedNotification, tips) } From 88c0d5f5adb03fc2856d886a4763f6fe17d157d2 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:38:14 +0800 Subject: [PATCH 06/91] feat: support incremental synchronization (#2379) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * group version * merge * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * fix: sort by id avoid unstable sort friends. * test: test log add. * test: debug log remove. * fix: transfer group owner incr version more than 1. * fix: add condition to kick owner. * feat: replace resp nil * feat: replace nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: nil * fix: delete cache of max group joined version avoid sync joined group failed. * fix: delete cache of max group joined version avoid sync joined group failed. * return group information for any changes * online cache --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- config/redis.yml | 2 +- go.mod | 6 +- go.sum | 8 +- internal/api/friend.go | 50 ++-- internal/api/group.go | 62 ++++ internal/api/router.go | 13 +- internal/push/push_handler.go | 2 +- internal/rpc/friend/black.go | 21 +- internal/rpc/friend/callback.go | 24 +- internal/rpc/friend/friend.go | 145 ++++++---- internal/rpc/friend/notification.go | 15 +- internal/rpc/friend/sync.go | 78 ++++++ internal/rpc/group/convert.go | 4 + internal/rpc/group/group.go | 85 ++++-- internal/rpc/group/notification.go | 42 ++- internal/rpc/group/sync.go | 149 ++++++++++ internal/rpc/incrversion/option.go | 156 +++++++++++ internal/rpc/msg/seq.go | 4 +- internal/rpc/user/user.go | 107 +++++-- pkg/common/cmd/group.go | 3 +- pkg/common/cmd/msg_gateway_test.go | 7 + pkg/common/convert/user.go | 28 +- pkg/common/storage/cache/cachekey/friend.go | 10 + pkg/common/storage/cache/cachekey/group.go | 10 + pkg/common/storage/cache/friend.go | 12 + pkg/common/storage/cache/group.go | 9 +- pkg/common/storage/cache/redis/friend.go | 60 +++- pkg/common/storage/cache/redis/group.go | 93 ++++-- pkg/common/storage/controller/friend.go | 55 +++- pkg/common/storage/controller/group.go | 149 +++++++--- pkg/common/storage/controller/user.go | 6 + pkg/common/storage/database/friend.go | 11 + pkg/common/storage/database/group.go | 4 + pkg/common/storage/database/group_member.go | 5 + pkg/common/storage/database/mgo/black.go | 2 +- .../storage/database/mgo/conversation.go | 3 +- pkg/common/storage/database/mgo/friend.go | 138 +++++++-- .../storage/database/mgo/friend_request.go | 2 +- pkg/common/storage/database/mgo/group.go | 35 ++- .../storage/database/mgo/group_member.go | 127 +++++++-- .../storage/database/mgo/group_request.go | 2 +- pkg/common/storage/database/mgo/log.go | 2 +- pkg/common/storage/database/mgo/object.go | 2 +- pkg/common/storage/database/mgo/user.go | 68 ++++- .../storage/database/mgo/version_log.go | 265 ++++++++++++++++++ .../storage/database/mgo/version_test.go | 39 +++ pkg/common/storage/database/name.go | 17 ++ pkg/common/storage/database/user.go | 3 + pkg/common/storage/database/version_log.go | 19 ++ pkg/common/storage/model/friend.go | 18 +- pkg/common/storage/model/user.go | 4 +- pkg/common/storage/model/version_log.go | 69 +++++ pkg/common/storage/versionctx/rpc.go | 14 + pkg/common/storage/versionctx/version.go | 48 ++++ pkg/rpcclient/friend.go | 20 +- pkg/util/hashutil/id.go | 16 ++ 56 files changed, 2023 insertions(+), 325 deletions(-) create mode 100644 internal/rpc/friend/sync.go create mode 100644 internal/rpc/group/sync.go create mode 100644 internal/rpc/incrversion/option.go create mode 100644 pkg/common/storage/database/mgo/version_log.go create mode 100644 pkg/common/storage/database/mgo/version_test.go create mode 100644 pkg/common/storage/database/name.go create mode 100644 pkg/common/storage/database/version_log.go create mode 100644 pkg/common/storage/model/version_log.go create mode 100644 pkg/common/storage/versionctx/rpc.go create mode 100644 pkg/common/storage/versionctx/version.go create mode 100644 pkg/util/hashutil/id.go diff --git a/config/redis.yml b/config/redis.yml index 6fe0dd02d4..87abed0e1c 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -3,4 +3,4 @@ username: '' password: openIM123 clusterMode: false db: 0 -maxRetry: 10 \ No newline at end of file +maxRetry: 10 diff --git a/go.mod b/go.mod index 5e17e866cd..245214b932 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.66-alpha.1 - github.com/openimsdk/tools v0.0.49-alpha.19 + github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/tools v0.0.49-alpha.28 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -176,3 +176,5 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 7c5e024495..664f2366a7 100644 --- a/go.sum +++ b/go.sum @@ -270,10 +270,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.66-alpha.1 h1:/8y+aXQeX6+IgfFxujHbRgJylqJRkwF5gMrwNhWMsiU= -github.com/openimsdk/protocol v0.0.66-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= -github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= +github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= +github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/friend.go b/internal/api/friend.go index 1fea38b313..11d7375fa3 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -17,7 +17,7 @@ package api import ( "github.com/gin-gonic/gin" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" ) @@ -28,68 +28,82 @@ func NewFriendApi(client rpcclient.Friend) FriendApi { } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.ApplyToAddFriend, o.Client, c) + a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.Call(friend.FriendClient.RespondFriendApply, o.Client, c) + a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.DeleteFriend, o.Client, c) + a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriendsApply, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) + //a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c, a2r.NewNilReplaceOption(relation.FriendClient.GetDesignatedFriends)) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.Call(friend.FriendClient.SetFriendRemark, o.Client, c) + a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.AddBlack, o.Client, c) + a2r.Call(relation.FriendClient.AddBlack, o.Client, c) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationBlacks, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.RemoveBlack, o.Client, c) + a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.ImportFriends, o.Client, c) + a2r.Call(relation.FriendClient.ImportFriends, o.Client, c) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.IsFriend, o.Client, c) + a2r.Call(relation.FriendClient.IsFriend, o.Client, c) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.Call(friend.FriendClient.GetFriendIDs, o.Client, c) + a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) + a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } + func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) + a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c) +} + +func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) +} + +func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) +} + +func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { + a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c) } diff --git a/internal/api/group.go b/internal/api/group.go index 6079c53437..e48191ee15 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -19,6 +19,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" + "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/log" ) type GroupApi rpcclient.Group @@ -65,6 +67,7 @@ func (o *GroupApi) GetGroupUsersReqApplicationList(c *gin.Context) { func (o *GroupApi) GetGroupsInfo(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c) + //a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo)) } func (o *GroupApi) KickGroupMember(c *gin.Context) { @@ -73,6 +76,7 @@ func (o *GroupApi) KickGroupMember(c *gin.Context) { func (o *GroupApi) GetGroupMembersInfo(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c) + //a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo)) } func (o *GroupApi) GetGroupMemberList(c *gin.Context) { @@ -134,3 +138,61 @@ func (o *GroupApi) GetGroups(c *gin.Context) { func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c) } + +func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { + type BatchIncrementalReq struct { + UserID string `json:"user_id"` + List []*group.GetIncrementalGroupMemberReq `json:"list"` + } + type BatchIncrementalResp struct { + List map[string]*group.GetIncrementalGroupMemberResp `json:"list"` + } + req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq](c) + if err != nil { + apiresp.GinError(c, err) + return + } + resp := &BatchIncrementalResp{ + List: make(map[string]*group.GetIncrementalGroupMemberResp), + } + var ( + changeCount int + ) + for _, req := range req.List { + if _, ok := resp.List[req.GroupID]; ok { + continue + } + res, err := o.Client.GetIncrementalGroupMember(c, req) + if err != nil { + if len(resp.List) == 0 { + apiresp.GinError(c, err) + } else { + log.ZError(c, "group incr sync versopn", err, "groupID", req.GroupID, "success", len(resp.List)) + apiresp.GinSuccess(c, resp) + } + return + } + resp.List[req.GroupID] = res + changeCount += len(res.Insert) + len(res.Delete) + len(res.Update) + if changeCount >= 200 { + break + } + } + apiresp.GinSuccess(c, resp) +} + +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) +} + +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 6005671785..0f46f26baf 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,9 @@ package api import ( "fmt" + "net/http" + "strings" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" @@ -14,8 +17,6 @@ import ( "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "net/http" - "strings" ) func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { @@ -81,11 +82,14 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/add_black", f.AddBlack) friendRouterGroup.POST("/get_black_list", f.GetPaginationBlacks) friendRouterGroup.POST("/remove_black", f.RemoveBlack) + friendRouterGroup.POST("/get_incremental_blacks", f.GetIncrementalBlacks) friendRouterGroup.POST("/import_friend", f.ImportFriends) friendRouterGroup.POST("/is_friend", f.IsFriend) friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) + friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) + friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") @@ -114,6 +118,11 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 03c299b7ab..dfe0e7b55d 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -180,7 +180,7 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat } func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get super group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + log.ZDebug(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) var pushToUserIDs []string if err = c.webhookBeforeGroupOnlinePush(ctx, &c.config.WebhooksConfig.BeforeGroupOnlinePush, groupID, msg, &pushToUserIDs); err != nil { diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index caec08b7ac..218d1e7f85 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -16,16 +16,17 @@ package friend import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) -func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.GetPaginationBlacksReq) (resp *pbfriend.GetPaginationBlacksResp, err error) { +func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.GetPaginationBlacksReq) (resp *relation.GetPaginationBlacksResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -33,7 +34,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge if err != nil { return nil, err } - resp = &pbfriend.GetPaginationBlacksResp{} + resp = &relation.GetPaginationBlacksResp{} resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -42,18 +43,18 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge return resp, nil } -func (s *friendServer) IsBlack(ctx context.Context, req *pbfriend.IsBlackReq) (*pbfriend.IsBlackResp, error) { +func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (*relation.IsBlackResp, error) { in1, in2, err := s.blackDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } - resp := &pbfriend.IsBlackResp{} + resp := &relation.IsBlackResp{} resp.InUser1Blacks = in1 resp.InUser2Blacks = in2 return resp, nil } -func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlackReq) (*pbfriend.RemoveBlackResp, error) { +func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlackReq) (*relation.RemoveBlackResp, error) { if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -64,10 +65,10 @@ func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlac s.notificationSender.BlackDeletedNotification(ctx, req) - return &pbfriend.RemoveBlackResp{}, nil + return &relation.RemoveBlackResp{}, nil } -func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) (*pbfriend.AddBlackResp, error) { +func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) (*relation.AddBlackResp, error) { if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -87,5 +88,5 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) return nil, err } s.notificationSender.BlackAddedNotification(ctx, req) - return &pbfriend.AddBlackResp{}, nil + return &relation.AddBlackResp{}, nil } diff --git a/internal/rpc/friend/callback.go b/internal/rpc/friend/callback.go index 0610cdb78a..746ad21fa1 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/friend/callback.go @@ -16,14 +16,15 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" ) -func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { +func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *relation.DeleteFriendReq) { cbReq := &cbapi.CallbackAfterDeleteFriendReq{ CallbackCommand: cbapi.CallbackAfterDeleteFriendCommand, OwnerUserID: req.OwnerUserID, @@ -32,7 +33,7 @@ func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *conf s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterDeleteFriendResp{}, after) } -func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ApplyToAddFriendReq) error { +func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *relation.ApplyToAddFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendCommand, @@ -50,7 +51,7 @@ func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *confi }) } -func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.ApplyToAddFriendReq) { +func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *relation.ApplyToAddFriendReq) { cbReq := &cbapi.CallbackAfterAddFriendReq{ CallbackCommand: cbapi.CallbackAfterAddFriendCommand, FromUserID: req.FromUserID, @@ -61,8 +62,7 @@ func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config. s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *pbfriend.SetFriendRemarkReq) { - +func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *relation.SetFriendRemarkReq) { cbReq := &cbapi.CallbackAfterSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackAfterSetFriendRemarkCommand, OwnerUserID: req.OwnerUserID, @@ -73,7 +73,7 @@ func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *c s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *pbfriend.ImportFriendReq) { +func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *relation.ImportFriendReq) { cbReq := &cbapi.CallbackAfterImportFriendsReq{ CallbackCommand: cbapi.CallbackAfterImportFriendsCommand, OwnerUserID: req.OwnerUserID, @@ -83,7 +83,7 @@ func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *con s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *pbfriend.RemoveBlackReq) { +func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *relation.RemoveBlackReq) { cbReq := &cbapi.CallbackAfterRemoveBlackReq{ CallbackCommand: cbapi.CallbackAfterRemoveBlackCommand, OwnerUserID: req.OwnerUserID, @@ -93,7 +93,7 @@ func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *confi s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *pbfriend.SetFriendRemarkReq) error { +func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *relation.SetFriendRemarkReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackBeforeSetFriendRemarkCommand, @@ -112,7 +112,7 @@ func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before }) } -func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *pbfriend.AddBlackReq) error { +func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *relation.AddBlackReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddBlackReq{ CallbackCommand: cbapi.CallbackBeforeAddBlackCommand, @@ -124,7 +124,7 @@ func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config }) } -func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *pbfriend.RespondFriendApplyReq) error { +func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *relation.RespondFriendApplyReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendAgreeReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendAgreeCommand, @@ -138,7 +138,7 @@ func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before * }) } -func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ImportFriendReq) error { +func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *relation.ImportFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeImportFriendsReq{ CallbackCommand: cbapi.CallbackBeforeImportFriendsCommand, diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 8b2dea995f..622e19f422 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -30,7 +31,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -40,7 +41,7 @@ import ( ) type friendServer struct { - friendDatabase controller.FriendDatabase + db controller.FriendDatabase blackDatabase controller.BlackDatabase userRpcClient *rpcclient.UserRpcClient notificationSender *FriendNotificationSender @@ -54,7 +55,7 @@ type Config struct { RpcConfig config.Friend RedisConfig config.Redis MongodbConfig config.Mongo - //ZookeeperConfig config.ZooKeeper + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks @@ -100,8 +101,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) // Register Friend server with refactored MongoDB and Redis integrations - pbfriend.RegisterFriendServer(server, &friendServer{ - friendDatabase: controller.NewFriendDatabase( + relation.RegisterFriendServer(server, &friendServer{ + db: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB, redis.GetRocksCacheOptions()), @@ -123,8 +124,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // ok. -func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) (resp *pbfriend.ApplyToAddFriendResp, err error) { - resp = &pbfriend.ApplyToAddFriendResp{} +func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.ApplyToAddFriendReq) (resp *relation.ApplyToAddFriendResp, err error) { + resp = &relation.ApplyToAddFriendResp{} if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -138,14 +139,14 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply return nil, err } - in1, in2, err := s.friendDatabase.CheckIn(ctx, req.FromUserID, req.ToUserID) + in1, in2, err := s.db.CheckIn(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } if in1 && in2 { return nil, servererrs.ErrRelationshipAlready.WrapMsg("already friends has f") } - if err = s.friendDatabase.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { + if err = s.db.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { return nil, err } s.notificationSender.FriendApplicationAddNotification(ctx, req) @@ -154,7 +155,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply } // ok. -func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFriendReq) (resp *pbfriend.ImportFriendResp, err error) { +func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFriendReq) (resp *relation.ImportFriendResp, err error) { if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -172,11 +173,11 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr return nil, err } - if err := s.friendDatabase.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { + if err := s.db.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { return nil, err } for _, userID := range req.FriendUserIDs { - s.notificationSender.FriendApplicationAgreedNotification(ctx, &pbfriend.RespondFriendApplyReq{ + s.notificationSender.FriendApplicationAgreedNotification(ctx, &relation.RespondFriendApplyReq{ FromUserID: req.OwnerUserID, ToUserID: userID, HandleResult: constant.FriendResponseAgree, @@ -184,12 +185,12 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr } s.webhookAfterImportFriends(ctx, &s.config.WebhooksConfig.AfterImportFriends, req) - return &pbfriend.ImportFriendResp{}, nil + return &relation.ImportFriendResp{}, nil } // ok. -func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.RespondFriendApplyReq) (resp *pbfriend.RespondFriendApplyResp, err error) { - resp = &pbfriend.RespondFriendApplyResp{} +func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.RespondFriendApplyReq) (resp *relation.RespondFriendApplyResp, err error) { + resp = &relation.RespondFriendApplyResp{} if err := authverify.CheckAccessV3(ctx, req.ToUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -204,7 +205,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res if err := s.webhookBeforeAddFriendAgree(ctx, &s.config.WebhooksConfig.BeforeAddFriendAgree, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - err := s.friendDatabase.AgreeFriendRequest(ctx, &friendRequest) + err := s.db.AgreeFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -212,7 +213,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { - err := s.friendDatabase.RefuseFriendRequest(ctx, &friendRequest) + err := s.db.RefuseFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -223,16 +224,16 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res } // ok. -func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFriendReq) (resp *pbfriend.DeleteFriendResp, err error) { - resp = &pbfriend.DeleteFriendResp{} +func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFriendReq) (resp *relation.DeleteFriendResp, err error) { + resp = &relation.DeleteFriendResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { + if err := s.db.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { return nil, err } s.notificationSender.FriendDeletedNotification(ctx, req) @@ -241,19 +242,19 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFri } // ok. -func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFriendRemarkReq) (resp *pbfriend.SetFriendRemarkResp, err error) { +func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFriendRemarkReq) (resp *relation.SetFriendRemarkResp, err error) { if err = s.webhookBeforeSetFriendRemark(ctx, &s.config.WebhooksConfig.BeforeSetFriendRemark, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - resp = &pbfriend.SetFriendRemarkResp{} + resp = &relation.SetFriendRemarkResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { + if err := s.db.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { return nil, err } s.webhookAfterSetFriendRemark(ctx, &s.config.WebhooksConfig.AfterSetFriendRemark, req) @@ -262,29 +263,40 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFri } // ok. -func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.GetDesignatedFriendsReq) (resp *pbfriend.GetDesignatedFriendsResp, err error) { - resp = &pbfriend.GetDesignatedFriendsResp{} +func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { + resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + friends, err := s.getFriend(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } - if resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap); err != nil { + return &relation.GetDesignatedFriendsResp{ + FriendsInfo: friends, + }, nil +} + +func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friendUserIDs []string) ([]*sdkws.FriendInfo, error) { + if len(friendUserIDs) == 0 { + return nil, nil + } + friends, err := s.db.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + if err != nil { return nil, err } - return resp, nil + return convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, - req *pbfriend.GetDesignatedFriendsApplyReq) (resp *pbfriend.GetDesignatedFriendsApplyResp, err error) { - friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) + req *relation.GetDesignatedFriendsApplyReq, +) (resp *relation.GetDesignatedFriendsApplyResp, err error) { + friendRequests, err := s.db.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } - resp = &pbfriend.GetDesignatedFriendsApplyResp{} + resp = &relation.GetDesignatedFriendsApplyResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -293,15 +305,15 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, } // Get received friend requests (i.e., those initiated by others). -func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyToReq) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) { +func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *relation.GetPaginationFriendsApplyToReq) (resp *relation.GetPaginationFriendsApplyToResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsApplyToResp{} + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -310,12 +322,12 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbf return resp, nil } -func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyFromReq) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) { - resp = &pbfriend.GetPaginationFriendsApplyFromResp{} +func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { + resp = &relation.GetPaginationFriendsApplyFromResp{} if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -328,24 +340,24 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *p } // ok. -func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) (resp *pbfriend.IsFriendResp, err error) { - resp = &pbfriend.IsFriendResp{} - resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) +func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { + resp = &relation.IsFriendResp{} + resp.InUser1Friends, resp.InUser2Friends, err = s.db.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } return resp, nil } -func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.GetPaginationFriendsReq) (resp *pbfriend.GetPaginationFriendsResp, err error) { +func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.GetPaginationFriendsReq) (resp *relation.GetPaginationFriendsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + total, friends, err := s.db.PageOwnerFriends(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsResp{} + resp = &relation.GetPaginationFriendsResp{} resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -354,19 +366,19 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G return resp, nil } -func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriendIDsReq) (resp *pbfriend.GetFriendIDsResp, err error) { +func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriendIDsReq) (resp *relation.GetFriendIDsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - resp = &pbfriend.GetFriendIDsResp{} - resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) + resp = &relation.GetFriendIDsResp{} + resp.FriendIDs, err = s.db.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } return resp, nil } -func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfriend.GetSpecifiedFriendsInfoReq) (*pbfriend.GetSpecifiedFriendsInfoResp, error) { +func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relation.GetSpecifiedFriendsInfoReq) (*relation.GetSpecifiedFriendsInfoResp, error) { if len(req.UserIDList) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDList is empty") } @@ -377,7 +389,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien if err != nil { return nil, err } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) if err != nil { return nil, err } @@ -391,8 +403,8 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien blackMap := datautil.SliceToMap(blacks, func(e *model.Black) string { return e.BlockUserID }) - resp := &pbfriend.GetSpecifiedFriendsInfoResp{ - Infos: make([]*pbfriend.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), + resp := &relation.GetSpecifiedFriendsInfoResp{ + Infos: make([]*relation.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), } for _, userID := range req.UserIDList { user := userMap[userID] @@ -401,7 +413,6 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } var friendInfo *sdkws.FriendInfo if friend := friendMap[userID]; friend != nil { - friendInfo = &sdkws.FriendInfo{ OwnerUserID: friend.OwnerUserID, Remark: friend.Remark, @@ -422,7 +433,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien Ex: black.Ex, } } - resp.Infos = append(resp.Infos, &pbfriend.GetSpecifiedFriendsInfoInfo{ + resp.Infos = append(resp.Infos, &relation.GetSpecifiedFriendsInfoInfo{ UserInfo: user, FriendInfo: friendInfo, BlackInfo: blackInfo, @@ -430,10 +441,11 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, - req *pbfriend.UpdateFriendsReq, -) (*pbfriend.UpdateFriendsResp, error) { + req *relation.UpdateFriendsReq, +) (*relation.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("friendIDList is empty") } @@ -441,7 +453,7 @@ func (s *friendServer) UpdateFriends( return nil, errs.ErrArgs.WrapMsg("friendIDList repeated") } - _, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + _, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } @@ -457,12 +469,27 @@ func (s *friendServer) UpdateFriends( if req.Ex != nil { val["ex"] = req.Ex.Value } - if err = s.friendDatabase.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { + if err = s.db.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { return nil, err } - resp := &pbfriend.UpdateFriendsResp{} + resp := &relation.UpdateFriendsResp{} s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } + +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *relation.GetIncrementalFriendsApplyToReq) (*relation.GetIncrementalFriendsApplyToResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *relation.GetIncrementalFriendsApplyFromReq) (*relation.GetIncrementalFriendsApplyFromResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *relation.GetIncrementalBlacksReq) (*relation.GetIncrementalBlacksResp, error) { + // TODO implement me + return nil, nil +} diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index 8089a9bdc8..ddee025bb6 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -16,6 +16,7 @@ package friend import ( "context" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -24,7 +25,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) @@ -127,7 +128,7 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) { +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, ToUserID: req.ToUserID, @@ -137,7 +138,7 @@ func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context. func (f *FriendNotificationSender) FriendApplicationAgreedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -148,7 +149,7 @@ func (f *FriendNotificationSender) FriendApplicationAgreedNotification( func (f *FriendNotificationSender) FriendApplicationRefusedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -182,7 +183,7 @@ func (f *FriendNotificationSender) FriendAddedNotification( return nil } -func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *pbfriend.DeleteFriendReq) { +func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *relation.DeleteFriendReq) { tips := sdkws.FriendDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.FriendUserID, @@ -204,14 +205,14 @@ func (f *FriendNotificationSender) FriendsInfoUpdateNotification(ctx context.Con f.Notification(ctx, toUserID, toUserID, constant.FriendsInfoUpdateNotification, &tips) } -func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *pbfriend.AddBlackReq) { +func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *relation.AddBlackReq) { tips := sdkws.BlackAddedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = req.OwnerUserID tips.FromToUserID.ToUserID = req.BlackUserID f.Notification(ctx, req.OwnerUserID, req.BlackUserID, constant.BlackAddedNotification, &tips) } -func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *pbfriend.RemoveBlackReq) { +func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *relation.RemoveBlackReq) { blackDeletedTips := sdkws.BlackDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.BlackUserID, diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go new file mode 100644 index 0000000000..684894609c --- /dev/null +++ b/internal/rpc/friend/sync.go @@ -0,0 +1,78 @@ +package friend + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/sdkws" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/relation" +) + +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(ctx, userID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(ctx, req.UserID, userID) + } + return &relation.NotificationUserInfoUpdateResp{}, nil +} + +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &relation.GetFullFriendUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: s.db.FindFriendIncrVersion, + CacheMaxVersion: s.db.FindMaxFriendVersionCache, + Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { + return s.getFriend(ctx, req.UserID, ids) + }, + ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, + Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/group/convert.go b/internal/rpc/group/convert.go index a75693904d..8026430c30 100644 --- a/internal/rpc/group/convert.go +++ b/internal/rpc/group/convert.go @@ -57,3 +57,7 @@ func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel InviterUserID: member.InviterUserID, } } + +func (s *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { + return s.groupMemberDB2PB(member, 0) +} diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index a9cea4ff22..e3d1d4dfe6 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -17,17 +17,18 @@ package group import ( "context" "fmt" + "math/big" + "math/rand" + "strconv" + "strings" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "math/big" - "math/rand" - "strconv" - "strings" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" @@ -132,13 +133,17 @@ func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro } groupIDs = append(groupIDs, member.GroupID) } + for _, groupID := range groupIDs { + if err := s.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } for _, groupID := range groupIDs { s.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) } if err = s.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { return nil, err } - return &pbgroup.NotificationUserInfoUpdateResp{}, nil } @@ -527,6 +532,14 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou if datautil.Contain(opUserID, req.KickedUserIDs...) { return nil, errs.ErrArgs.WrapMsg("opUserID in KickedUserIDs") } + owner, err := s.db.TakeGroupOwner(ctx, req.GroupID) + if err != nil { + return nil, err + } + if datautil.Contain(owner.UserID, req.KickedUserIDs...) { + return nil, errs.ErrArgs.WrapMsg("ownerUID can not Kick") + } + members, err := s.db.FindGroupMembers(ctx, req.GroupID, append(req.KickedUserIDs, opUserID)) if err != nil { return nil, err @@ -586,7 +599,7 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou FaceURL: group.FaceURL, OwnerUserID: ownerUserID, CreateTime: group.CreateTime.UnixMilli(), - MemberCount: num, + MemberCount: num - uint32(len(req.KickedUserIDs)), Ex: group.Ex, Status: group.Status, CreatorUserID: group.CreatorUserID, @@ -621,18 +634,29 @@ func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, req.UserIDs) + members, err := s.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupMembersInfoResp{ + Members: members, + }, nil +} + +func (s *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + members, err := s.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } if err := s.PopulateGroupMember(ctx, members...); err != nil { return nil, err } - return &pbgroup.GetGroupMembersInfoResp{ - Members: datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { - return convert.Db2PbGroupMember(e) - }), - }, nil + return datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { + return convert.Db2PbGroupMember(e) + }), nil } // GetGroupApplicationList handles functions that get a list of group requests. @@ -701,15 +725,28 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupID is empty") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := s.getGroupsInfo(ctx, req.GroupIDs) if err != nil { return nil, err } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, req.GroupIDs) + return &pbgroup.GetGroupsInfoResp{ + GroupInfos: groups, + }, nil +} + +func (s *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { + if len(groupIDs) == 0 { + return nil, nil + } + groups, err := s.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, req.GroupIDs) + groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + if err != nil { + return nil, err + } + owners, err := s.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } @@ -719,15 +756,13 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - return &pbgroup.GetGroupsInfoResp{ - GroupInfos: datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { - var ownerUserID string - if owner, ok := ownerMap[e.GroupID]; ok { - ownerUserID = owner.UserID - } - return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) - }), - }, nil + return datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { + var ownerUserID string + if owner, ok := ownerMap[e.GroupID]; ok { + ownerUserID = owner.UserID + } + return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) + }), nil } func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index a9abb03e6a..a8824962dc 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -17,13 +17,15 @@ package group import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" @@ -293,6 +295,17 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + return + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -303,6 +316,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -316,6 +330,7 @@ func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -329,6 +344,7 @@ func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } @@ -342,6 +358,7 @@ func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -386,6 +403,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -469,14 +487,20 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. } opUserID := mcontext.GetOpUserID(ctx) var member map[string]*sdkws.GroupMemberFullInfo - member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID}) + member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID, req.OldOwnerUserID}) if err != nil { return } - tips := &sdkws.GroupOwnerTransferredTips{Group: group, OpUser: member[opUserID], NewGroupOwner: member[req.NewOwnerUserID]} + tips := &sdkws.GroupOwnerTransferredTips{ + Group: group, + OpUser: member[opUserID], + NewGroupOwner: member[req.NewOwnerUserID], + OldGroupOwnerInfo: member[req.OldOwnerUserID], + } if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -490,6 +514,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } @@ -513,6 +538,7 @@ func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, } tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) } @@ -534,6 +560,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return } tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) } @@ -574,6 +601,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -598,6 +626,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -625,6 +654,7 @@ func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, gr if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMutedNotification, tips) } @@ -652,6 +682,7 @@ func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupCancelMutedNotification, tips) } @@ -676,6 +707,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -699,6 +731,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -723,5 +756,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go new file mode 100644 index 0000000000..75d060c0e5 --- /dev/null +++ b/internal/rpc/group/sync.go @@ -0,0 +1,149 @@ +package group + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/constant" + pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" + "slices" +) + +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { + vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &pbgroup.GetFullGroupMemberUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { + vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(groupIDs) + if req.IdHash == idHash { + groupIDs = nil + } + return &pbgroup.GetFullJoinGroupIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + GroupIDs: groupIDs, + }, nil +} + +func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + group, err := s.db.TakeGroup(ctx, req.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } + var hasGroupUpdate bool + opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit) + if err != nil { + return nil, err + } + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == "" { + vl.LogLen-- + hasGroupUpdate = true + return true + } + return false + }) + if vl.LogLen > 0 { + hasGroupUpdate = true + } + return vl, nil + }, + CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, + Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + return s.getGroupMembersInfo(ctx, req.GroupID, ids) + }, + ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + return &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + resp, err := opt.Build() + if err != nil { + return nil, err + } + if resp.Full || hasGroupUpdate { + count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + resp.Group = s.groupDB2PB(group, owner.UserID, count) + } + return resp, nil +} + +func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: s.db.FindJoinIncrVersion, + CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, + Find: s.getGroupsInfo, + ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + return &pbgroup.GetIncrementalJoinGroupResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go new file mode 100644 index 0000000000..f7a71244a0 --- /dev/null +++ b/internal/rpc/incrversion/option.go @@ -0,0 +1,156 @@ +package incrversion + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +//func Limit(maxSync int, version uint64) int { +// if version == 0 { +// return 0 +// } +// return maxSync +//} + +const syncLimit = 200 + +const ( + tagQuery = iota + 1 + tagFull + tageEqual +) + +type Option[A, B any] struct { + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + //SyncLimit int + CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) + Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + //SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, deleteIds []string, insertList, updateList []A, full bool) *B +} + +func (o *Option[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *Option[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") + } + if o.VersionKey == "" { + return o.newError("versionKey is empty") + } + //if o.SyncLimit <= 0 { + // return o.newError("invalid synchronization quantity") + //} + if o.Version == nil { + return o.newError("func version is nil") + } + //if o.SortID == nil { + // return o.newError("func allID is nil") + //} + if o.Find == nil { + return o.newError("func find is nil") + } + if o.ID == nil { + return o.newError("func id is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil +} + +func (o *Option[A, B]) validVersion() bool { + objID, err := primitive.ObjectIDFromHex(o.VersionID) + return err == nil && (!objID.IsZero()) && o.VersionNumber > 0 +} + +func (o *Option[A, B]) equalID(objID primitive.ObjectID) bool { + return o.VersionID == objID.Hex() +} + +func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { + if o.CacheMaxVersion == nil { + if o.validVersion() { + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) + } + *tag = tagFull + return o.Version(o.Ctx, o.VersionKey, 0, 0) + } else { + cache, err := o.CacheMaxVersion(o.Ctx, o.VersionKey) + if err != nil { + return nil, err + } + if !o.validVersion() { + *tag = tagFull + return cache, nil + } + if !o.equalID(cache.ID) { + *tag = tagFull + return cache, nil + } + if o.VersionNumber == uint64(cache.Version) { + *tag = tageEqual + return cache, nil + } + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) + } +} + +func (o *Option[A, B]) Build() (*B, error) { + if err := o.check(); err != nil { + return nil, err + } + var tag int + version, err := o.getVersion(&tag) + if err != nil { + return nil, err + } + var full bool + switch tag { + case tagQuery: + full = version.ID.Hex() != o.VersionID || uint64(version.Version) < o.VersionNumber || len(version.Logs) != version.LogLen + case tagFull: + full = true + case tageEqual: + full = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } + var ( + insertIds []string + deleteIds []string + updateIds []string + ) + if !full { + insertIds, deleteIds, updateIds = version.DeleteAndChangeIDs() + } + var ( + insertList []A + updateList []A + ) + if len(insertIds) > 0 { + insertList, err = o.Find(o.Ctx, insertIds) + if err != nil { + return nil, err + } + } + if len(updateIds) > 0 { + updateList, err = o.Find(o.Ctx, updateIds) + if err != nil { + return nil, err + } + } + return o.Resp(version, deleteIds, insertList, updateList, full), nil +} diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 27465c2105..1ebec4a719 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -16,13 +16,15 @@ package msg import ( "context" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" pbmsg "github.com/openimsdk/protocol/msg" ) func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil { + if err != nil && errs.Unwrap(err) != redis.Nil { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index d0d3dbf603..211b360b7b 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,6 +16,7 @@ package user import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -23,9 +24,12 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/protocol/group" + friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" "math/rand" "strings" + "sync" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -131,26 +135,29 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -164,25 +171,29 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - } + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -683,3 +694,45 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } + +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} + +func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { + users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) + if err != nil { + return nil, err + } + return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil +} diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f158b8c626..20124be957 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/rpc/group" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -58,5 +59,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start) + a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg_gateway_test.go b/pkg/common/cmd/msg_gateway_test.go index d820627b50..2b68a3e3ab 100644 --- a/pkg/common/cmd/msg_gateway_test.go +++ b/pkg/common/cmd/msg_gateway_test.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/utils/jsonutil" "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson/primitive" "math" "testing" ) @@ -59,3 +60,9 @@ func TestName(t *testing.T) { t.Logf("%+v\n", rReso) } + +func TestName1(t *testing.T) { + t.Log(primitive.NewObjectID().String()) + t.Log(primitive.NewObjectID().Hex()) + +} diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index ccc574f51b..d824fa68e0 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -16,26 +16,26 @@ package convert import ( relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" "time" "github.com/openimsdk/protocol/sdkws" ) -func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { - result := make([]*sdkws.UserInfo, 0, len(users)) - for _, user := range users { - userPb := &sdkws.UserInfo{ - UserID: user.UserID, - Nickname: user.Nickname, - FaceURL: user.FaceURL, - Ex: user.Ex, - CreateTime: user.CreateTime.UnixMilli(), - AppMangerLevel: user.AppMangerLevel, - GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, - } - result = append(result, userPb) +func UserDB2Pb(user *relationtb.User) *sdkws.UserInfo { + return &sdkws.UserInfo{ + UserID: user.UserID, + Nickname: user.Nickname, + FaceURL: user.FaceURL, + Ex: user.Ex, + CreateTime: user.CreateTime.UnixMilli(), + AppMangerLevel: user.AppMangerLevel, + GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, } - return result +} + +func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { + return datautil.Slice(users, UserDB2Pb) } func UserPb2DB(user *sdkws.UserInfo) *relationtb.User { diff --git a/pkg/common/storage/cache/cachekey/friend.go b/pkg/common/storage/cache/cachekey/friend.go index 9691b1f5c9..8a053ca324 100644 --- a/pkg/common/storage/cache/cachekey/friend.go +++ b/pkg/common/storage/cache/cachekey/friend.go @@ -19,6 +19,8 @@ const ( TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" FriendKey = "FRIEND_INFO:" IsFriendKey = "IS_FRIEND:" // local cache key + //FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendMaxVersionKey = "FRIEND_MAX_VERSION:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -33,6 +35,14 @@ func GetFriendKey(ownerUserID, friendUserID string) string { return FriendKey + ownerUserID + "-" + friendUserID } +func GetFriendMaxVersionKey(ownerUserID string) string { + return FriendMaxVersionKey + ownerUserID +} + func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } + +//func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { +// return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +//} diff --git a/pkg/common/storage/cache/cachekey/group.go b/pkg/common/storage/cache/cachekey/group.go index 681121ecba..2ef42c0ff4 100644 --- a/pkg/common/storage/cache/cachekey/group.go +++ b/pkg/common/storage/cache/cachekey/group.go @@ -28,6 +28,8 @@ const ( JoinedGroupsKey = "JOIN_GROUPS_KEY:" GroupMemberNumKey = "GROUP_MEMBER_NUM_CACHE:" GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:" + GroupMemberMaxVersionKey = "GROUP_MEMBER_MAX_VERSION:" + GroupJoinMaxVersionKey = "GROUP_JOIN_MAX_VERSION:" ) func GetGroupInfoKey(groupID string) string { @@ -57,3 +59,11 @@ func GetGroupMemberNumKey(groupID string) string { func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string { return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel)) } + +func GetGroupMemberMaxVersionKey(groupID string) string { + return GroupMemberMaxVersionKey + groupID +} + +func GetJoinGroupMaxVersionKey(userID string) string { + return GroupJoinMaxVersionKey + userID +} diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index acff829f86..b451d36757 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -32,4 +32,16 @@ type FriendCache interface { DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + + DelMaxFriendVersion(ownerUserIDs ...string) FriendCache + + //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + + FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 53e2cd1c74..73479bb1bb 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -46,7 +46,6 @@ type GroupCache interface { GetGroupMemberInfo(ctx context.Context, groupID, userID string) (groupMember *model.GroupMember, err error) GetGroupMembersInfo(ctx context.Context, groupID string, userID []string) (groupMembers []*model.GroupMember, err error) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) - GetGroupMembersPage(ctx context.Context, groupID string, userID []string, showNumber, pageNumber int32) (total uint32, groupMembers []*model.GroupMember, err error) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) @@ -59,4 +58,12 @@ type GroupCache interface { GetGroupRolesLevelMemberInfo(ctx context.Context, groupID string, roleLevels []int32) ([]*model.GroupMember, error) GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache + + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + DelMaxGroupMemberVersion(groupIDs ...string) GroupCache + DelMaxJoinGroupVersion(userIDs ...string) GroupCache + FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index f76e5ff6ba..01988310c9 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -16,6 +16,8 @@ package redis import ( "context" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -25,7 +27,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -38,6 +39,7 @@ type FriendCacheRedis struct { friendDB database.Friend expireTime time.Duration rcClient *rockscache.Client + syncCount int } // NewFriendCacheRedis creates a new instance of FriendCacheRedis. @@ -68,6 +70,14 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } +//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { +// return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +//} + +func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { + return cachekey.GetFriendMaxVersionKey(ownerUserID) +} + // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string { return cachekey.GetTwoWayFriendsIDsKey(ownerUserID) @@ -97,6 +107,16 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } +//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { +// newGroupCache := f.CloneFriendCache() +// keys := make([]string, 0, len(ownerUserIDs)) +// for _, userID := range ownerUserIDs { +// keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) +// } +// newGroupCache.AddKeys(keys...) +// return newGroupCache +//} + // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { friendIDs, err := f.GetFriendIDs(ctx, ownerUserID) @@ -151,3 +171,41 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } + +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendKey(ownerUserID, friendUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + +func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendMaxVersionKey(ownerUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + +//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// userIDs, err := f.GetFriendIDs(ctx, ownerUserID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > f.syncCount { +// userIDs = userIDs[:f.syncCount] +// } +// return userIDs, nil +//} + +func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) + }) +} diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 2de03906f1..589678c506 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -27,7 +27,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" ) @@ -111,6 +110,14 @@ func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLeve return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel) } +func (g *GroupCacheRedis) getGroupMemberMaxVersionKey(groupID string) string { + return cachekey.GetGroupMemberMaxVersionKey(groupID) +} + +func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { + return cachekey.GetJoinGroupMaxVersionKey(userID) +} + func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { key := g.getGroupInfoKey(group.GroupID) for i, _key := range keys { @@ -246,9 +253,17 @@ func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { return cache } +func (g *GroupCacheRedis) findUserJoinedGroupID(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + if err != nil { + return nil, err + } + return g.groupDB.FindJoinSortGroupID(ctx, groupIDs) +} + func (g *GroupCacheRedis) GetJoinedGroupIDs(ctx context.Context, userID string) (joinedGroupIDs []string, err error) { return getCache(ctx, g.rcClient, g.getJoinedGroupsKey(userID), g.expireTime, func(ctx context.Context) ([]string, error) { - return g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + return g.findUserJoinedGroupID(ctx, userID) }) } @@ -277,26 +292,6 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin }) } -func (g *GroupCacheRedis) GetGroupMembersPage( - ctx context.Context, - groupID string, - userIDs []string, - showNumber, pageNumber int32, -) (total uint32, groupMembers []*model.GroupMember, err error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return 0, nil, err - } - if userIDs != nil { - userIDs = datautil.BothExist(userIDs, groupMemberIDs) - } else { - userIDs = groupMemberIDs - } - groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber))) - - return uint32(len(userIDs)), groupMembers, err -} - func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) { groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) if err != nil { @@ -406,3 +401,57 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st return g.groupMemberDB.Take(ctx, groupID, userID) }) } + +//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := g.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > g.syncCount { +// userIDs = userIDs[:g.syncCount] +// } +// return userIDs, nil +//} +// +//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { +// groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) +// if err != nil { +// return nil, err +// } +// if len(groupIDs) > g.syncCount { +// groupIDs = groupIDs[:g.syncCount] +// } +// return groupIDs, nil +//} + +func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(groupIDs)) + for _, groupID := range groupIDs { + keys = append(keys, g.getGroupMemberMaxVersionKey(groupID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + keys = append(keys, g.getJoinGroupMaxVersionKey(userID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, 0, 0) + }) +} + +func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 1c3d9f139a..e402f5980b 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -77,6 +77,16 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + + FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } type friendDatabase struct { @@ -175,7 +185,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return err } newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...) + cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...) return cache.ChainExecDel(ctx) }) @@ -278,7 +288,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelMaxFriendVersion(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -287,7 +297,8 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).ChainExecDel(ctx) + userIds := append(friendUserIDs, ownerUserID) + return f.cache.DelFriendIDs(userIds...).DelMaxFriendVersion(userIds...).ChainExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -295,7 +306,7 @@ func (f *friendDatabase) UpdateRemark(ctx context.Context, ownerUserID, friendUs if err := f.friend.UpdateRemark(ctx, ownerUserID, friendUserID, remark); err != nil { return err } - return f.cache.DelFriend(ownerUserID, friendUserID).ChainExecDel(ctx) + return f.cache.DelFriend(ownerUserID, friendUserID).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } // PageOwnerFriends retrieves the list of friends for the ownerUserID. It does not return an error if the result is empty. @@ -324,9 +335,6 @@ func (f *friendDatabase) FindFriendsWithError(ctx context.Context, ownerUserID s if err != nil { return } - if len(friends) != len(friendUserIDs) { - err = errs.ErrRecordNotFound.Wrap() - } return } @@ -341,8 +349,37 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if len(val) == 0 { return nil } - if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return err + } + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) + }) +} + +//func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +//} + +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) +} + +func (f *friendDatabase) FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return f.cache.FindMaxFriendVersion(ctx, ownerUserID) +} + +func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + return f.friend.FindFriendUserID(ctx, friendUserID) +} + +//func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +//} + +func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + if err := f.friend.IncrVersion(ctx, ownerUserID, friendUserIDs, state); err != nil { return err } - return f.cache.DelFriends(ownerUserID, friendUserIDs).ChainExecDel(ctx) + return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f2a1358357..3a5f48d4c6 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -106,6 +106,20 @@ type GroupDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) // DeleteGroupMemberHash deletes the hash entries for group members in specified groups. DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error + + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error + + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) + + SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) + + FindJoinGroupID(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -134,6 +148,10 @@ type groupDatabase struct { cache cache.GroupCache } +func (g *groupDatabase) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) { + return g.cache.GetJoinedGroupIDs(ctx, userID) +} + func (g *groupDatabase) FindGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { return g.cache.GetGroupMembersInfo(ctx, groupID, userIDs) } @@ -174,7 +192,8 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMembersHash(group.GroupID). DelGroupsMemberNum(group.GroupID). DelGroupMemberIDs(group.GroupID). - DelGroupAllRoleLevel(group.GroupID) + DelGroupAllRoleLevel(group.GroupID). + DelMaxGroupMemberVersion(group.GroupID) } } if len(groupMembers) > 0 { @@ -187,7 +206,9 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMemberIDs(groupMember.GroupID). DelJoinedGroupID(groupMember.UserID). DelGroupMembersInfo(groupMember.GroupID, groupMember.UserID). - DelGroupAllRoleLevel(groupMember.GroupID) + DelGroupAllRoleLevel(groupMember.GroupID). + DelMaxJoinGroupVersion(groupMember.UserID). + DelMaxGroupMemberVersion(groupMember.GroupID) } } return c.ChainExecDel(ctx) @@ -219,10 +240,15 @@ func (g *groupDatabase) SearchGroup(ctx context.Context, keyword string, paginat } func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data map[string]any) error { - if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { - return err - } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { + return err + } + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { + return err + } + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) + }) } func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error { @@ -244,7 +270,19 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupsMemberNum(groupID). DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). - DelGroupMembersInfo(groupID, userIDs...) + DelGroupMembersInfo(groupID, userIDs...). + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...) + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + } else { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { + return err + } + c = c.DelMaxGroupMemberVersion(groupID) } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) @@ -316,7 +354,9 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, DelGroupMemberIDs(groupID). DelGroupsMemberNum(groupID). DelJoinedGroupID(member.UserID). - DelGroupRoleLevel(groupID, []int32{member.RoleLevel}) + DelGroupRoleLevel(groupID, []int32{member.RoleLevel}). + DelMaxJoinGroupVersion(userID). + DelMaxGroupMemberVersion(groupID) if err := c.ChainExecDel(ctx); err != nil { return err } @@ -326,17 +366,21 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, } func (g *groupDatabase) DeleteGroupMember(ctx context.Context, groupID string, userIDs []string) error { - if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { - return err - } - c := g.cache.CloneGroupCache() - return c.DelGroupMembersHash(groupID). - DelGroupMemberIDs(groupID). - DelGroupsMemberNum(groupID). - DelJoinedGroupID(userIDs...). - DelGroupMembersInfo(groupID, userIDs...). - DelGroupAllRoleLevel(groupID). - ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { + return err + } + c := g.cache.CloneGroupCache() + return c.DelGroupMembersHash(groupID). + DelGroupMemberIDs(groupID). + DelGroupsMemberNum(groupID). + DelJoinedGroupID(userIDs...). + DelGroupMembersInfo(groupID, userIDs...). + DelGroupAllRoleLevel(groupID). + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...). + ChainExecDel(ctx) + }) } func (g *groupDatabase) MapGroupMemberUserID(ctx context.Context, groupIDs []string) (map[string]*common.GroupSimpleUserID, error) { @@ -357,29 +401,35 @@ func (g *groupDatabase) MapGroupMemberNum(ctx context.Context, groupIDs []string func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, oldOwnerUserID, newOwnerUserID string, roleLevel int32) error { return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { - if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, oldOwnerUserID, roleLevel); err != nil { - return err - } - if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, newOwnerUserID, constant.GroupOwner); err != nil { + if err := g.groupMemberDB.UpdateUserRoleLevels(ctx, groupID, oldOwnerUserID, roleLevel, newOwnerUserID, constant.GroupOwner); err != nil { return err } c := g.cache.CloneGroupCache() return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). - DelGroupMembersHash(groupID).ChainExecDel(ctx) + DelGroupMembersHash(groupID). + DelMaxGroupMemberVersion(groupID). + DelGroupMemberIDs(groupID). + ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { - if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { - return err - } - c := g.cache.CloneGroupCache() - c = c.DelGroupMembersInfo(groupID, userID) - if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) + if len(data) == 0 { + return nil } - return c.ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { + return err + } + c := g.cache.CloneGroupCache() + c = c.DelGroupMembersInfo(groupID, userID) + if g.groupMemberDB.IsUpdateRoleLevel(data) { + c = c.DelGroupAllRoleLevel(groupID).DelGroupMemberIDs(groupID) + } + c = c.DelMaxGroupMemberVersion(groupID) + return c.ChainExecDel(ctx) + }) } func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.BatchUpdateGroupMember) error { @@ -390,9 +440,9 @@ func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.B return err } if g.groupMemberDB.IsUpdateRoleLevel(item.Map) { - c = c.DelGroupAllRoleLevel(item.GroupID) + c = c.DelGroupAllRoleLevel(item.GroupID).DelGroupMemberIDs(item.GroupID) } - c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelGroupMembersHash(item.GroupID) + c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelMaxGroupMemberVersion(item.GroupID).DelGroupMembersHash(item.GroupID) } return c.ChainExecDel(ctx) }) @@ -443,3 +493,34 @@ func (g *groupDatabase) DeleteGroupMemberHash(ctx context.Context, groupIDs []st } return c.ChainExecDel(ctx) } + +func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit) +} + +func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) +} + +func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { + return g.cache.FindMaxGroupMemberVersion(ctx, groupID) +} + +func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { + return g.cache.FindMaxJoinGroupVersion(ctx, userID) +} + +func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + groupIDs, err := g.cache.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return 0, nil, err + } + return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) +} + +func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, userIDs, state); err != nil { + return err + } + return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 09dc2db22b..9efe535c01 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -60,6 +60,8 @@ type UserDatabase interface { CountTotal(ctx context.Context, before *time.Time) (int64, error) // CountRangeEverydayTotal Get the user increment in the range CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) // SubscribeUsersStatus Subscribe a user's presence status SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error // UnsubscribeUsersStatus unsubscribe a user's presence status @@ -210,6 +212,10 @@ func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.T return u.userDB.CountRangeEverydayTotal(ctx, start, end) } +func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + return u.userDB.SortQuery(ctx, userIDName, asc) +} + // SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 33d9c17bc5..b596411fce 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -46,4 +47,14 @@ type Friend interface { FindFriendUserIDs(ctx context.Context, ownerUserID string) (friendUserIDs []string, err error) // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) + + IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } diff --git a/pkg/common/storage/database/group.go b/pkg/common/storage/database/group.go index 712db09d2d..7ef22f6c9a 100644 --- a/pkg/common/storage/database/group.go +++ b/pkg/common/storage/database/group.go @@ -32,4 +32,8 @@ type Group interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get Group total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) + + SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index f57f2c3173..c272b6ef6f 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -25,6 +25,7 @@ type GroupMember interface { Delete(ctx context.Context, groupID string, userIDs []string) (err error) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error + UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error) @@ -34,4 +35,8 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/black.go b/pkg/common/storage/database/mgo/black.go index cf74cfab14..4a7a35e6f1 100644 --- a/pkg/common/storage/database/mgo/black.go +++ b/pkg/common/storage/database/mgo/black.go @@ -27,7 +27,7 @@ import ( ) func NewBlackMongo(db *mongo.Database) (database.Black, error) { - coll := db.Collection("black") + coll := db.Collection(database.BlackName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 9c35f841b6..b462d39583 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" @@ -29,7 +30,7 @@ import ( ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { - coll := db.Collection("conversation") + coll := db.Collection(database.ConversationName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index ffa006d013..7f456fbdab 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -18,6 +18,8 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" @@ -28,12 +30,13 @@ import ( // FriendMgo implements Friend using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection + coll *mongo.Collection + owner database.VersionLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. func NewFriendMongo(db *mongo.Database) (database.Friend, error) { - coll := db.Collection("friend") + coll := db.Collection(database.FriendName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, @@ -44,12 +47,41 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - return &FriendMgo{coll: coll}, nil + owner, err := NewVersionLog(db.Collection(database.FriendVersionName)) + if err != nil { + return nil, err + } + return &FriendMgo{coll: coll, owner: owner}, nil +} + +func (f *FriendMgo) friendSort() any { + return bson.D{{"is_pinned", -1}, {"_id", 1}} } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { - return mongoutil.InsertMany(ctx, f.coll, friends) + for i, friend := range friends { + if friend.ID.IsZero() { + friends[i].ID = primitive.NewObjectID() + } + if friend.CreateTime.IsZero() { + friends[i].CreateTime = time.Now() + } + } + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, f.coll, friends) + }, func() error { + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) + } + for ownerUserID, friendUserIDs := range mp { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } // Delete removes specified friends of the owner user. @@ -58,11 +90,15 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.DeleteOne(ctx, f.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteOne(ctx, f.coll, filter) + }, func() error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete) + }) } // UpdateByMap updates specific fields of a friend document using a map. -func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]interface{}) error { +func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]any) error { if len(args) == 0 { return nil } @@ -70,30 +106,55 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + }, func() error { + return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate) + }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) } +func (f *FriendMgo) fillTime(friends ...*model.Friend) { + for i, friend := range friends { + if friend.CreateTime.IsZero() { + friends[i].CreateTime = friend.ID.Timestamp() + } + } +} + +func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) { + friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friend) + return friend, nil +} + +func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) { + friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friends...) + return friends, nil +} + +func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) { + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...) +} + // Take retrieves a single friend document. Returns an error if not found. func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) { filter := bson.M{ "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + return f.findOne(ctx, filter) } // FindUserState finds the friendship status between two users. @@ -104,7 +165,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string) {"owner_user_id": userID2, "friend_user_id": userID1}, }, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error. @@ -113,7 +174,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindReversalFriends finds users who have added the specified user as a friend. @@ -122,25 +183,33 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string "owner_user_id": bson.M{"$in": ownerUserIDs}, "friend_user_id": friendUserID, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) +} + +func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { + filter := bson.M{"owner_user_id": ownerUserID} + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit)) + return mongoutil.Find[string](ctx, f.coll, filter, opt) } // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"friend_user_id": friendUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) } // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { @@ -158,7 +227,24 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - // Perform the update operation for all matching documents - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, update) - return err + return mongoutil.IncrVersion(func() error { + return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) + }, func() error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) + }) +} + +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) +} + +func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + filter := bson.M{ + "friend_user_id": friendUserID, + } + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) +} + +func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) } diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 0d60b213dd..4eed2f4a26 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -27,7 +27,7 @@ import ( ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { - coll := db.Collection("friend_request") + coll := db.Collection(database.FriendRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "from_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 48d24560bc..3be7883af9 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -30,7 +30,7 @@ import ( ) func NewGroupMongo(db *mongo.Database) (database.Group, error) { - coll := db.Collection("group") + coll := db.Collection(database.GroupName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -47,6 +47,10 @@ type GroupMgo struct { coll *mongo.Collection } +func (g *GroupMgo) sortGroup() any { + return bson.D{{"group_name", 1}, {"create_time", 1}} +} + func (g *GroupMgo) Create(ctx context.Context, groups []*model.Group) (err error) { return mongoutil.InsertMany(ctx, g.coll, groups) } @@ -126,3 +130,32 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (g *GroupMgo) FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) { + if len(groupIDs) < 2 { + return groupIDs, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + opt := options.Find().SetSort(g.sortGroup()).SetProjection(bson.M{"_id": 0, "group_id": 1}) + return mongoutil.Find[string](ctx, g.coll, filter, opt) +} + +func (g *GroupMgo) SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + if len(groupIDs) == 0 { + return 0, nil, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + if keyword != "" { + filter["group_name"] = bson.M{"$regex": keyword} + } + // Define the sorting options + opts := options.Find().SetSort(g.sortGroup()) + // Perform the search with pagination and sorting + return mongoutil.FindPage[*model.Group](ctx, g.coll, filter, pagination, opts) +} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ccca386e57..3eb93a10ec 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/log" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" @@ -29,7 +30,7 @@ import ( ) func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { - coll := db.Collection("group_member") + coll := db.Collection(database.GroupMemberName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -40,15 +41,53 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - return &GroupMemberMgo{coll: coll}, nil + member, err := NewVersionLog(db.Collection(database.GroupMemberVersionName)) + if err != nil { + return nil, err + } + join, err := NewVersionLog(db.Collection(database.GroupJoinVersionName)) + if err != nil { + return nil, err + } + return &GroupMemberMgo{coll: coll, member: member, join: join}, nil } type GroupMemberMgo struct { - coll *mongo.Collection + coll *mongo.Collection + member database.VersionLog + join database.VersionLog +} + +func (g *GroupMemberMgo) memberSort() any { + return bson.D{{"role_level", -1}, {"create_time", -1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { - return mongoutil.InsertMany(ctx, g.coll, groupMembers) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, g.coll, groupMembers) + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.GroupID] = append(gms[member.GroupID], member.UserID) + } + for groupID, userIDs := range gms { + if err := g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.UserID] = append(gms[member.UserID], member.GroupID) + } + for userID, groupIDs := range gms { + if err := g.join.IncrVersion(ctx, userID, groupIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) { @@ -56,24 +95,62 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s if len(userIDs) > 0 { filter["user_id"] = bson.M{"$in": userIDs} } - return mongoutil.DeleteMany(ctx, g.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteMany(ctx, g.coll, filter) + }, func() error { + if len(userIDs) == 0 { + return g.member.Delete(ctx, groupID) + } else { + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + } + }, func() error { + for _, userID := range userIDs { + if err := g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error { - return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, + bson.M{"$set": bson.M{"role_level": roleLevel}}, true) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + }) } - -func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) { - return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) +func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error { + return mongoutil.IncrVersion(func() error { + if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": firstUserID}, + bson.M{"$set": bson.M{"role_level": firstUserRoleLevel}}, true); err != nil { + return err + } + if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": secondUserID}, + bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil { + return err + } + + return nil + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{firstUserID, secondUserID}, model.VersionStateUpdate) + }) } -func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*model.GroupMember, err error) { - // TODO implement me - panic("implement me") +func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) { + if len(data) == 0 { + return nil + } + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + }) } func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -88,13 +165,13 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) } -func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { +func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (int64, []*model.GroupMember, error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.memberSort())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { @@ -118,3 +195,21 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { _, ok := data["role_level"] return ok } + +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error { + return g.join.IncrVersion(ctx, userID, groupIDs, state) +} + +func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + return g.member.IncrVersion(ctx, groupID, userIDs, state) +} + +func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find member incr version", "groupID", groupID, "version", version) + return g.member.FindChangeLog(ctx, groupID, version, limit) +} + +func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version) + return g.join.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index 4ae7785276..b1942b7083 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -28,7 +28,7 @@ import ( ) func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { - coll := db.Collection("group_request") + coll := db.Collection(database.GroupRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/log.go b/pkg/common/storage/database/mgo/log.go index 51715bd771..6ff4c60395 100644 --- a/pkg/common/storage/database/mgo/log.go +++ b/pkg/common/storage/database/mgo/log.go @@ -28,7 +28,7 @@ import ( ) func NewLogMongo(db *mongo.Database) (database.Log, error) { - coll := db.Collection("log") + coll := db.Collection(database.LogName) _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ { Keys: bson.D{ diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 8ed7b3a561..df4d10ec4f 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -27,7 +27,7 @@ import ( ) func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { - coll := db.Collection("s3") + coll := db.Collection(database.ObjectName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "name", Value: 1}, diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 96cb18882b..8978e64ebf 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -31,7 +31,7 @@ import ( ) func NewUserMongo(db *mongo.Database) (database.User, error) { - coll := db.Collection("user") + coll := db.Collection(database.UserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, @@ -319,3 +319,69 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (u *UserMgo) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + if len(userIDName) == 0 { + return nil, nil + } + userIDs := make([]string, 0, len(userIDName)) + attached := make(map[string]string) + for userID, name := range userIDName { + userIDs = append(userIDs, userID) + if name == "" { + continue + } + attached[userID] = name + } + var sortValue int + if asc { + sortValue = 1 + } else { + sortValue = -1 + } + if len(attached) == 0 { + filter := bson.M{"user_id": bson.M{"$in": userIDs}} + opt := options.Find().SetSort(bson.M{"nickname": sortValue}) + return mongoutil.Find[*model.User](ctx, u.coll, filter, opt) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": bson.M{"$in": userIDs}, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$arrayElemAt": []any{ + bson.M{ + "$filter": bson.M{ + "input": bson.M{ + "$objectToArray": attached, + }, + "as": "item", + "cond": bson.M{ + "$eq": []any{"$$item.k", "$user_id"}, + }, + }, + }, + 0, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$ifNull": []any{"$_query_sort_name.v", "$nickname"}, + }, + }, + }, + { + "$sort": bson.M{ + "_query_sort_name": sortValue, + }, + }, + } + return mongoutil.Aggregate[*model.User](ctx, u.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go new file mode 100644 index 0000000000..8836742f08 --- /dev/null +++ b/pkg/common/storage/database/mgo/version_log.go @@ -0,0 +1,265 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { + lm := &VersionLogMgo{coll: coll} + if lm.initIndex(context.Background()) != nil { + return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) + } + return lm, nil +} + +type VersionLogMgo struct { + coll *mongo.Collection +} + +func (l *VersionLogMgo) initIndex(ctx context.Context) error { + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + }) + return err +} + +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { + _, err := l.IncrVersionResult(ctx, dId, eIds, state) + return err +} + +func (l *VersionLogMgo) IncrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + vl, err := l.incrVersionResult(ctx, dId, eIds, state) + if err != nil { + return nil, err + } + versionctx.GetVersionLog(ctx).Append(versionctx.Collection{ + Name: l.coll.Name(), + Doc: vl, + }) + return vl, nil +} + +func (l *VersionLogMgo) incrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + now := time.Now() + if res, err := l.writeLogBatch2(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + if res, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !mongo.IsDuplicateKeyError(err) { + return nil, err + } + return l.writeLogBatch2(ctx, dId, eIds, state, now) +} + +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { + wl := model.VersionLogTable{ + ID: primitive.NewObjectID(), + DID: dId, + Logs: make([]model.VersionLogElem, 0, len(eIds)), + Version: database.FirstVersion, + Deleted: database.DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, model.VersionLogElem{ + EID: eId, + State: state, + Version: database.FirstVersion, + LastUpdate: now, + }) + } + if _, err := l.coll.InsertOne(ctx, &wl); err != nil { + return nil, err + } + return wl.VersionLog(), nil +} + +func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { + if eIds == nil { + eIds = []string{} + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "state": state, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0}) + return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) +} + +func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { + vl, err := mongoutil.FindOne[*model.VersionLogTable](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err != nil { + return nil, err + } + return vl.VersionLog(), nil +} + +func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { + if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { + return wl, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + log.ZDebug(ctx, "init doc", "dId", dId) + if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dId) + return res, nil + } else if mongo.IsDuplicateKeyError(err) { + return l.findChangeLog(ctx, dId, version, limit) + } else { + return nil, err + } +} + +func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { + if version == 0 && limit == 0 { + return l.findDoc(ctx, dId) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": dId, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + vl, err := mongoutil.Aggregate[*model.VersionLog](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(vl) == 0 { + return nil, mongo.ErrNoDocuments + } + return vl[0], nil +} + +func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) +} + +func (l *VersionLogMgo) Delete(ctx context.Context, dId string) error { + return mongoutil.DeleteOne(ctx, l.coll, bson.M{"d_id": dId}) +} diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go new file mode 100644 index 0000000000..236c61a2ce --- /dev/null +++ b/pkg/common/storage/database/mgo/version_test.go @@ -0,0 +1,39 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("version_test") + tmp, err := NewVersionLog(coll) + if err != nil { + panic(err) + } + vl := tmp.(*VersionLogMgo) + res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) + if err != nil { + t.Log(err) + return + } + t.Logf("%+v", res) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go new file mode 100644 index 0000000000..986f22a1a2 --- /dev/null +++ b/pkg/common/storage/database/name.go @@ -0,0 +1,17 @@ +package database + +const ( + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" +) diff --git a/pkg/common/storage/database/user.go b/pkg/common/storage/database/user.go index 2e40886205..4ddc8285f6 100644 --- a/pkg/common/storage/database/user.go +++ b/pkg/common/storage/database/user.go @@ -39,6 +39,9 @@ type User interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get user total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) + // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go new file mode 100644 index 0000000000..9d7bcc1724 --- /dev/null +++ b/pkg/common/storage/database/version_log.go @@ -0,0 +1,19 @@ +package database + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "time" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type VersionLog interface { + IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error + Delete(ctx context.Context, dId string) error +} diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2a..abcca2f2b9 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -15,17 +15,19 @@ package model import ( + "go.mongodb.org/mongo-driver/bson/primitive" "time" ) // Friend represents the data structure for a friend relationship in MongoDB. type Friend struct { - OwnerUserID string `bson:"owner_user_id"` - FriendUserID string `bson:"friend_user_id"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` + ID primitive.ObjectID `bson:"_id"` + OwnerUserID string `bson:"owner_user_id"` + FriendUserID string `bson:"friend_user_id"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` } diff --git a/pkg/common/storage/model/user.go b/pkg/common/storage/model/user.go index c6a4f952c2..f64d09e797 100644 --- a/pkg/common/storage/model/user.go +++ b/pkg/common/storage/model/user.go @@ -36,10 +36,10 @@ func (u *User) GetFaceURL() string { return u.FaceURL } -func (u User) GetUserID() string { +func (u *User) GetUserID() string { return u.UserID } -func (u User) GetEx() string { +func (u *User) GetEx() string { return u.Ex } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go new file mode 100644 index 0000000000..11a40ef24f --- /dev/null +++ b/pkg/common/storage/model/version_log.go @@ -0,0 +1,69 @@ +package model + +import ( + "context" + "errors" + "github.com/openimsdk/tools/log" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +const ( + VersionStateInsert = iota + 1 + VersionStateDelete + VersionStateUpdate +) + +type VersionLogElem struct { + EID string `bson:"e_id"` + State int32 `bson:"state"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type VersionLogTable struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (v *VersionLogTable) VersionLog() *VersionLog { + return &VersionLog{ + ID: v.ID, + DID: v.DID, + Logs: v.Logs, + Version: v.Version, + Deleted: v.Deleted, + LastUpdate: v.LastUpdate, + LogLen: len(v.Logs), + } +} + +type VersionLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` +} + +func (v *VersionLog) DeleteAndChangeIDs() (insertIds, deleteIds, updateIds []string) { + for _, l := range v.Logs { + switch l.State { + case VersionStateInsert: + insertIds = append(insertIds, l.EID) + case VersionStateDelete: + deleteIds = append(deleteIds, l.EID) + case VersionStateUpdate: + updateIds = append(updateIds, l.EID) + default: + log.ZError(context.Background(), "invalid version status found", errors.New("dirty database data"), "objID", v.ID.Hex(), "did", v.DID, "elem", l) + } + } + return +} diff --git a/pkg/common/storage/versionctx/rpc.go b/pkg/common/storage/versionctx/rpc.go new file mode 100644 index 0000000000..67b95aebd5 --- /dev/null +++ b/pkg/common/storage/versionctx/rpc.go @@ -0,0 +1,14 @@ +package versionctx + +import ( + "context" + "google.golang.org/grpc" +) + +func EnableVersionCtx() grpc.ServerOption { + return grpc.ChainUnaryInterceptor(enableVersionCtxInterceptor) +} + +func enableVersionCtxInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(WithVersionLog(ctx), req) +} diff --git a/pkg/common/storage/versionctx/version.go b/pkg/common/storage/versionctx/version.go new file mode 100644 index 0000000000..5db8856401 --- /dev/null +++ b/pkg/common/storage/versionctx/version.go @@ -0,0 +1,48 @@ +package versionctx + +import ( + "context" + tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "sync" +) + +type Collection struct { + Name string + Doc *tablerelation.VersionLog +} + +type versionKey struct{} + +func WithVersionLog(ctx context.Context) context.Context { + return context.WithValue(ctx, versionKey{}, &VersionLog{}) +} + +func GetVersionLog(ctx context.Context) *VersionLog { + if v, ok := ctx.Value(versionKey{}).(*VersionLog); ok { + return v + } + return nil +} + +type VersionLog struct { + lock sync.Mutex + data []Collection +} + +func (v *VersionLog) Append(data ...Collection) { + if v == nil || len(data) == 0 { + return + } + v.lock.Lock() + defer v.lock.Unlock() + v.data = append(v.data, data...) +} + +func (v *VersionLog) Get() []Collection { + if v == nil { + return nil + } + v.lock.Lock() + defer v.lock.Unlock() + return v.data +} diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go index 5543afe4f7..fd00be3292 100644 --- a/pkg/rpcclient/friend.go +++ b/pkg/rpcclient/friend.go @@ -17,7 +17,7 @@ package rpcclient import ( "context" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" sdkws "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/system/program" @@ -26,7 +26,7 @@ import ( type Friend struct { conn grpc.ClientConnInterface - Client friend.FriendClient + Client relation.FriendClient discov discovery.SvcDiscoveryRegistry } @@ -35,7 +35,7 @@ func NewFriend(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *F if err != nil { program.ExitWithError(err) } - client := friend.NewFriendClient(conn) + client := relation.NewFriendClient(conn) return &Friend{discov: discov, conn: conn, Client: client} } @@ -47,11 +47,11 @@ func NewFriendRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName s func (f *FriendRpcClient) GetFriendsInfo( ctx context.Context, - ownerUserID, friendUserID string, + ownerUserID, relationUserID string, ) (resp *sdkws.FriendInfo, err error) { r, err := f.Client.GetDesignatedFriends( ctx, - &friend.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{friendUserID}}, + &relation.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{relationUserID}}, ) if err != nil { return nil, err @@ -60,17 +60,17 @@ func (f *FriendRpcClient) GetFriendsInfo( return } -// possibleFriendUserID Is PossibleFriendUserId's friends. +// possibleFriendUserID Is PossibleFriendUserId's relations. func (f *FriendRpcClient) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (bool, error) { - resp, err := f.Client.IsFriend(ctx, &friend.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) + resp, err := f.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) if err != nil { return false, err } return resp.InUser1Friends, nil } -func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) { - req := friend.GetFriendIDsReq{UserID: ownerUserID} +func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (relationIDs []string, err error) { + req := relation.GetFriendIDsReq{UserID: ownerUserID} resp, err := f.Client.GetFriendIDs(ctx, &req) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) } func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := b.Client.IsBlack(ctx, &friend.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) + r, err := b.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) if err != nil { return false, err } diff --git a/pkg/util/hashutil/id.go b/pkg/util/hashutil/id.go new file mode 100644 index 0000000000..52e7f4c6f2 --- /dev/null +++ b/pkg/util/hashutil/id.go @@ -0,0 +1,16 @@ +package hashutil + +import ( + "crypto/md5" + "encoding/binary" + "encoding/json" +) + +func IdHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} From 95df4194ca47a0f99e0f0bbf06a9263965c93d5a Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:25:10 +0800 Subject: [PATCH 07/91] Optimize get conversation seq (#2387) * feat:optimize GetConversationsHasReadAndMaxSeq * fix:get max seqs * fix:get max seqs * fix:get max seqs * fix:get max seqs --- pkg/common/storage/cache/redis/seq.go | 39 ++++++++++++++++++++++---- pkg/rpccache/conversation.go | 40 +++++++++++++++++++++------ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go index 76dd921a50..09ad5b6095 100644 --- a/pkg/common/storage/cache/redis/seq.go +++ b/pkg/common/storage/cache/redis/seq.go @@ -16,11 +16,13 @@ package redis import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/stringutil" "github.com/redis/go-redis/v9" + "sync" ) func NewSeqCache(rdb redis.UniversalClient) cache.SeqCache { @@ -61,15 +63,40 @@ func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey fun func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) { m = make(map[string]int64, len(items)) + var ( + reverseMap = make(map[string]string, len(items)) + keys = make([]string, len(items)) + lock sync.Mutex + ) + for i, v := range items { - res, err := c.rdb.Get(ctx, getkey(v)).Result() - if err != nil && err != redis.Nil { - return nil, errs.Wrap(err) + keys[i] = getkey(v) + reverseMap[getkey(v)] = v + } + + manager := NewRedisShardManager(c.rdb) + if err = manager.ProcessKeysBySlot(ctx, keys, func(ctx context.Context, _ int64, keys []string) error { + res, err := c.rdb.MGet(ctx, keys...).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return errs.Wrap(err) } - val := stringutil.StringToInt64(res) - if val != 0 { - m[items[i]] = val + + // len(res) <= len(items) + for i := range res { + strRes, ok := res[i].(string) + if !ok { + continue + } + val := stringutil.StringToInt64(strRes) + if val != 0 { + lock.Lock() + m[reverseMap[keys[i]]] = val + lock.Unlock() + } } + return nil + }); err != nil { + return nil, err } return m, nil diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 4c00dd1f7e..0109f1b1dd 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -16,15 +16,19 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" + "golang.org/x/sync/errgroup" +) + +const ( + conversationWorkerCount = 20 ) func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { @@ -90,15 +94,33 @@ func (c *ConversationLocalCache) GetSingleConversationRecvMsgOpt(ctx context.Con } func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { - conversations := make([]*pbconversation.Conversation, 0, len(conversationIDs)) + var ( + conversations = make([]*pbconversation.Conversation, 0, len(conversationIDs)) + conversationsChan = make(chan *pbconversation.Conversation, len(conversationIDs)) + ) + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(conversationWorkerCount) + for _, conversationID := range conversationIDs { - conversation, err := c.GetConversation(ctx, ownerUserID, conversationID) - if err != nil { - if errs.ErrRecordNotFound.Is(err) { - continue + conversationID := conversationID + g.Go(func() error { + conversation, err := c.GetConversation(ctx, ownerUserID, conversationID) + if err != nil { + if errs.ErrRecordNotFound.Is(err) { + return nil + } + return err } - return nil, err - } + conversationsChan <- conversation + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + close(conversationsChan) + for conversation := range conversationsChan { conversations = append(conversations, conversation) } return conversations, nil From 644eaf996c8ae0875bcd2a257621c97b821ce61b Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:27:16 +0800 Subject: [PATCH 08/91] update gomake version (#2386) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version --------- Co-authored-by: withchao --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f765805c3..e082dd64c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.13 +RUN go get github.com/openimsdk/gomake@v0.0.14-alpha.5 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/go.mod b/go.mod index 245214b932..365933de65 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.13 + github.com/openimsdk/gomake v0.0.14-alpha.5 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible diff --git a/go.sum b/go.sum index 664f2366a7..a5a620a144 100644 --- a/go.sum +++ b/go.sum @@ -268,8 +268,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= -github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= +github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= From f231ea1f2132b96a91d333c467cabb7832cb8260 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:39:59 +0800 Subject: [PATCH 09/91] fix:start (#2389) --- pkg/common/startrpc/start.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 069c92012a..b531daa47d 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -158,7 +158,6 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC } return nil case <-netDone: - close(netDone) return netErr } } From cc2f993eab4eeb39f69524b2257d35da6bb1b182 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 4 Jul 2024 19:58:41 +0800 Subject: [PATCH 10/91] fix:log (#2396) --- config/log.yml | 12 +++++++++--- go.mod | 5 ++--- go.sum | 14 ++++---------- pkg/common/cmd/root.go | 5 ++++- pkg/common/config/config.go | 21 ++++++++++++--------- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/config/log.yml b/config/log.yml index 2194d89174..38926479c7 100644 --- a/config/log.yml +++ b/config/log.yml @@ -1,9 +1,15 @@ # Log storage path, default is acceptable, change to a full path if modification is needed storageLocation: ../../../../logs/ # Log rotation period (in hours), default is acceptable -rotationTime: 24 -# Number of log files to retain, default is acceptable -remainRotationCount: 2 +rotationTime: 1 +# Maximum size of each log file (in MB), default is acceptable, it means unlimited +maxSize: 0 +# Number of log files to retain, default is acceptable, it means whenever the log file is rotated, the old log file will be deleted +maxBackups: 10 +# Old log retain time (in days), default is acceptable, it means unlimited +maxAge: 10 +# Whether compress old log files. +compress: false # Log level settings: 3 for production environment; 6 for more verbose logging in debugging environments remainLogLevel: 6 # Whether to output to standard output, default is acceptable diff --git a/go.mod b/go.mod index 365933de65..2111208f29 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,9 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.17 - github.com/openimsdk/tools v0.0.49-alpha.28 + github.com/openimsdk/tools v0.0.49-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -161,6 +160,7 @@ require ( google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gorm.io/gorm v1.25.8 // indirect stathat.com/c/consistent v1.0.0 // indirect ) @@ -168,7 +168,6 @@ require ( require ( github.com/go-playground/locales v0.14.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/spf13/cobra v1.8.0 github.com/ugorji/go/codec v1.2.11 // indirect diff --git a/go.sum b/go.sum index a5a620a144..5bc9f6d984 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -222,12 +220,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= -github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= @@ -272,8 +264,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= -github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= +github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -527,6 +519,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 08bb6d0642..f4fcf680bc 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -136,8 +136,11 @@ func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { r.log.IsStdout, r.log.IsJson, r.log.StorageLocation, - r.log.RemainRotationCount, r.log.RotationTime, + r.log.MaxBackups, + r.log.MaxSize, + r.log.MaxAge, + r.log.Compress, config.Version, ) if err != nil { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 0b6176fb74..536e4d5474 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -41,13 +41,16 @@ type LocalCache struct { } type Log struct { - StorageLocation string `mapstructure:"storageLocation"` - RotationTime uint `mapstructure:"rotationTime"` - RemainRotationCount uint `mapstructure:"remainRotationCount"` - RemainLogLevel int `mapstructure:"remainLogLevel"` - IsStdout bool `mapstructure:"isStdout"` - IsJson bool `mapstructure:"isJson"` - WithStack bool `mapstructure:"withStack"` + StorageLocation string `mapstructure:"storageLocation"` + RotationTime uint `mapstructure:"rotationTime"` + RemainLogLevel int `mapstructure:"remainLogLevel"` + MaxSize int `mapstructure:"maxSize"` + MaxBackups int `mapstructure:"maxBackups"` + MaxAge int `mapstructure:"maxAge"` + Compress bool `mapstructure:"compress"` + IsStdout bool `mapstructure:"isStdout"` + IsJson bool `mapstructure:"isJson"` + WithStack bool `mapstructure:"withStack"` } type Minio struct { @@ -339,8 +342,8 @@ type BeforeConfig struct { } type AfterConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` + Enable bool `mapstructure:"enable"` + Timeout int `mapstructure:"timeout"` AttentionIds []string `mapstructure:"attentionIds"` } From 407a117a05170f5d5855e2fa4e27d5dad2c86aa6 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 8 Jul 2024 12:34:46 +0800 Subject: [PATCH 11/91] fix: remove repeat append logic. (#2400) --- pkg/common/storage/database/mgo/msg.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index a7291fcc8f..04aa59cef9 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -3,10 +3,11 @@ package mgo import ( "context" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/utils/datautil" - "time" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -358,7 +359,6 @@ func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) ( msgInfo.Msg.ContentType = constant.MsgRevokeNotification msgInfo.Msg.Content = string(content) } - msgs = append(msgs, msgInfo) } //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber //n := int32(len(msgs)) From 213613cf5473fad8200ef4b5f10b3b1703fd45ca Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:17:44 +0800 Subject: [PATCH 12/91] Fix token (#2403) * fix:log config * fix: token update expire time --- config/log.yml | 2 +- pkg/common/storage/controller/auth.go | 16 ++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/config/log.yml b/config/log.yml index 38926479c7..1f5641622d 100644 --- a/config/log.yml +++ b/config/log.yml @@ -1,7 +1,7 @@ # Log storage path, default is acceptable, change to a full path if modification is needed storageLocation: ../../../../logs/ # Log rotation period (in hours), default is acceptable -rotationTime: 1 +rotationTime: 24 # Maximum size of each log file (in MB), default is acceptable, it means unlimited maxSize: 0 # Number of log files to retain, default is acceptable, it means whenever the log file is rotated, the old log file will be deleted diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index fbfe30836a..b725513d9e 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -55,7 +55,6 @@ func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, p // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - isCreate := true // flag is create or update tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID) if err != nil { return "", err @@ -66,9 +65,6 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI if err != nil || v != constant.NormalToken { deleteTokenKey = append(deleteTokenKey, k) } - if v == constant.NormalToken { - isCreate = false - } } if len(deleteTokenKey) != 0 { err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) @@ -84,16 +80,8 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", errs.WrapMsg(err, "token.SignedString") } - if isCreate { - // should create,should specify expiration time - if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err - } - } else { - // should update - if err = a.cache.SetTokenFlag(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err - } + if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { + return "", err } return tokenString, nil } From 28898f5b793195a59f0024b5ed3362ed3c57e5c5 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 10 Jul 2024 15:48:12 +0800 Subject: [PATCH 13/91] feat: implement server-initiated heartbeat in msgGateway module (#2404) * feat: implement send ping msg when platform is web in gateway. * add context life cycle control. * feat: implement heartbeat logic in msggateway. * update heartbeat logic. * update to correct method name and comment. * update initiate heartbeat logic. * rename ws_server * update writePingMsg logic * update log level to warn. --- internal/msggateway/client.go | 53 +++++++++++++++++++ internal/msggateway/constant.go | 3 ++ internal/msggateway/long_conn.go | 3 +- .../{n_ws_server.go => ws_server.go} | 0 4 files changed, 58 insertions(+), 1 deletion(-) rename internal/msggateway/{n_ws_server.go => ws_server.go} (100%) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 0581a025b4..1270eb9781 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -20,6 +20,7 @@ import ( "runtime/debug" "sync" "sync/atomic" + "time" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" @@ -72,6 +73,8 @@ type Client struct { closed atomic.Bool closedErr error token string + hbCtx context.Context + hbCancel context.CancelFunc } // ResetClient updates the client's state with new connection and context information. @@ -88,6 +91,7 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closed.Store(false) c.closedErr = nil c.token = ctx.GetToken() + c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) } func (c *Client) pingHandler(_ string) error { @@ -98,6 +102,13 @@ func (c *Client) pingHandler(_ string) error { return c.writePongMsg() } +func (c *Client) pongHandler(_ string) error { + if err := c.conn.SetReadDeadline(pongWait); err != nil { + return err + } + return nil +} + // readMessage continuously reads messages from the connection. func (c *Client) readMessage() { defer func() { @@ -110,7 +121,9 @@ func (c *Client) readMessage() { c.conn.SetReadLimit(maxMessageSize) _ = c.conn.SetReadDeadline(pongWait) + c.conn.SetPongHandler(c.pongHandler) c.conn.SetPingHandler(c.pingHandler) + c.activeHeartbeat(c.hbCtx) for { log.ZDebug(c.ctx, "readMessage") @@ -147,6 +160,7 @@ func (c *Client) readMessage() { case CloseMessage: c.closedErr = ErrClientClosed return + default: } } @@ -235,6 +249,7 @@ func (c *Client) close() { c.closed.Store(true) c.conn.Close() + c.hbCancel() // Close server-initiated heartbeat. c.longConnServer.UnRegister(c) } @@ -321,6 +336,44 @@ func (c *Client) writeBinaryMsg(resp Resp) error { return c.conn.WriteMessage(MessageBinary, encodedBuf) } +// Actively initiate Heartbeat when platform in Web. +func (c *Client) activeHeartbeat(ctx context.Context) { + if c.PlatformID == constant.WebPlatformID { + go func() { + log.ZDebug(ctx, "server initiative send heartbeat start.") + ticker := time.NewTicker(pingPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := c.writePingMsg(); err != nil { + log.ZWarn(c.ctx, "send Ping Message error.", err) + return + } + case <-c.hbCtx.Done(): + return + } + } + }() + } +} +func (c *Client) writePingMsg() error { + if c.closed.Load() { + return nil + } + + c.w.Lock() + defer c.w.Unlock() + + err := c.conn.SetWriteDeadline(writeWait) + if err != nil { + return err + } + + return c.conn.WriteMessage(PingMessage, nil) +} + func (c *Client) writePongMsg() error { if c.closed.Load() { return nil diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 64664ac0ab..125be16358 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -53,6 +53,9 @@ const ( // Time allowed to read the next pong message from the peer. pongWait = 30 * time.Second + // Send pings to peer with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + // Maximum message size allowed from peer. maxMessageSize = 51200 ) diff --git a/internal/msggateway/long_conn.go b/internal/msggateway/long_conn.go index 7d5bef4c3a..c1b3e27c93 100644 --- a/internal/msggateway/long_conn.go +++ b/internal/msggateway/long_conn.go @@ -16,10 +16,11 @@ package msggateway import ( "encoding/json" - "github.com/openimsdk/tools/apiresp" "net/http" "time" + "github.com/openimsdk/tools/apiresp" + "github.com/gorilla/websocket" "github.com/openimsdk/tools/errs" ) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/ws_server.go similarity index 100% rename from internal/msggateway/n_ws_server.go rename to internal/msggateway/ws_server.go From ea7e505269b748f2dbb757b6450e79f0a41a1721 Mon Sep 17 00:00:00 2001 From: printlin <32053356+printlin@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:10:04 +0800 Subject: [PATCH 14/91] feature: support for Kodo (#2363) * support for Kodo * fix mod bug --- config/openim-rpc-third.yml | 8 ++++++++ go.mod | 19 +++++++++++++++++++ internal/rpc/third/third.go | 3 +++ pkg/common/config/config.go | 33 ++++++++++++++++++++++++--------- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index bde38ccc44..6fb60f47f8 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -29,4 +29,12 @@ object: accessKeyID: '' accessKeySecret: '' sessionToken: '' + publicRead: false + kodo: + endpoint: "http://s3.cn-south-1.qiniucs.com" + bucket: "kodo-bucket-test" + bucketURL: "http://kodo-bucket-test-oetobfb.qiniudns.com" + accessKeyID: '' + accessKeySecret: '' + sessionToken: '' publicRead: false \ No newline at end of file diff --git a/go.mod b/go.mod index 2111208f29..79af68dbca 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,24 @@ require ( cloud.google.com/go/longrunning v0.5.4 // indirect cloud.google.com/go/storage v1.36.0 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect + github.com/aws/aws-sdk-go-v2 v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect + github.com/aws/smithy-go v1.17.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -118,6 +136,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/qiniu/go-sdk/v7 v7.18.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 7560486a00..b371f5e4e7 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -33,6 +33,7 @@ import ( "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" + "github.com/openimsdk/tools/s3/kodo" "google.golang.org/grpc" ) @@ -81,6 +82,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": o, err = oss.NewOSS(*config.RpcConfig.Object.Oss.Build()) + case "kodo": + o, err = kodo.NewKodo(*config.RpcConfig.Object.Kodo.Build()) default: err = fmt.Errorf("invalid object enable: %s", enable) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 536e4d5474..d8916ad04c 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -21,6 +21,7 @@ import ( "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" + "github.com/openimsdk/tools/s3/kodo" "strings" "time" ) @@ -281,15 +282,7 @@ type Third struct { Enable string `mapstructure:"enable"` Cos Cos `mapstructure:"cos"` Oss Oss `mapstructure:"oss"` - Kodo struct { - Endpoint string `mapstructure:"endpoint"` - Bucket string `mapstructure:"bucket"` - BucketURL string `mapstructure:"bucketURL"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` - } `mapstructure:"kodo"` + Kodo Kodo `mapstructure:"kodo"` Aws struct { Endpoint string `mapstructure:"endpoint"` Region string `mapstructure:"region"` @@ -317,6 +310,16 @@ type Oss struct { PublicRead bool `mapstructure:"publicRead"` } +type Kodo struct { + Endpoint string `mapstructure:"endpoint"` + Bucket string `mapstructure:"bucket"` + BucketURL string `mapstructure:"bucketURL"` + AccessKeyID string `mapstructure:"accessKeyID"` + AccessKeySecret string `mapstructure:"accessKeySecret"` + SessionToken string `mapstructure:"sessionToken"` + PublicRead bool `mapstructure:"publicRead"` +} + type User struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` @@ -528,6 +531,18 @@ func (o *Oss) Build() *oss.Config { } } +func (o *Kodo) Build() *kodo.Config { + return &kodo.Config{ + Endpoint: o.Endpoint, + Bucket: o.Bucket, + BucketURL: o.BucketURL, + AccessKeyID: o.AccessKeyID, + AccessKeySecret: o.AccessKeySecret, + SessionToken: o.SessionToken, + PublicRead: o.PublicRead, + } +} + func (l *CacheConfig) Failed() time.Duration { return time.Second * time.Duration(l.FailedExpire) } From 5f52fa19bd01cdbe3a387ec273546cd996ffddfd Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:35:27 +0800 Subject: [PATCH 15/91] feat: incremental synchronization of session list (#2408) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * conversation version incremental * GetOwnerConversation * fix: change incremental syncer router name. * fix: GetMsgDocModelByIndex bug * update go.mod --------- Co-authored-by: withchao Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> --- go.mod | 4 +- go.sum | 77 ++++++++++++++++++- internal/api/conversation.go | 12 +++ internal/api/friend.go | 3 + internal/api/router.go | 14 ++-- internal/rpc/conversation/conversaion.go | 25 +++++- internal/rpc/conversation/sync.go | 56 ++++++++++++++ internal/rpc/third/s3.go | 5 ++ internal/rpc/third/third.go | 1 + internal/rpc/user/user.go | 5 ++ .../storage/cache/cachekey/conversation.go | 5 ++ pkg/common/storage/cache/conversation.go | 4 + .../storage/cache/redis/conversation.go | 17 ++++ pkg/common/storage/controller/conversation.go | 35 ++++++++- pkg/common/storage/database/conversation.go | 2 +- .../storage/database/mgo/conversation.go | 65 ++++++++++++---- pkg/common/storage/database/mgo/msg.go | 2 +- pkg/common/storage/database/name.go | 27 +++---- 18 files changed, 314 insertions(+), 45 deletions(-) create mode 100644 internal/rpc/conversation/sync.go diff --git a/go.mod b/go.mod index 79af68dbca..207c2cf5e7 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 - github.com/openimsdk/tools v0.0.49-alpha.45 + github.com/openimsdk/protocol v0.0.69-alpha.30 + github.com/openimsdk/tools v0.0.49-alpha.49 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 5bc9f6d984..ef8f68a912 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,42 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= +github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= +github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= +github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 h1:1w11lfXOa8HoHoSlNtt4mqv/N3HmDOa+OnUH3Y9DHm8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -49,6 +85,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -95,12 +132,18 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -214,10 +257,16 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= @@ -262,14 +311,15 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= -github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= +github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.49 h1:bkVxlEDM9rJxo5a98mk2ETR0xbEv+5mjeVVVFQfARAQ= +github.com/openimsdk/tools v0.0.49-alpha.49/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -286,12 +336,18 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -324,6 +380,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -396,7 +453,9 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= @@ -423,6 +482,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= @@ -445,20 +505,26 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -515,8 +581,10 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -527,6 +595,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= diff --git a/internal/api/conversation.go b/internal/api/conversation.go index f273eaa4a7..360313ea87 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -50,3 +50,15 @@ func (o *ConversationApi) SetConversations(c *gin.Context) { func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { a2r.Call(conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client, c) } + +func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client, c) +} + +func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetIncrementalConversation, o.Client, c) +} + +func (o *ConversationApi) GetOwnerConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetOwnerConversation, o.Client, c) +} diff --git a/internal/api/friend.go b/internal/api/friend.go index 11d7375fa3..6c42184230 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -16,6 +16,7 @@ package api import ( "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" @@ -100,6 +101,8 @@ func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) } +// GetIncrementalBlacks is temporarily unused. +// Deprecated: This function is currently unused and may be removed in future versions. func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 0f46f26baf..822e647932 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -8,6 +8,9 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -15,8 +18,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { @@ -118,9 +119,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) - groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) - groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_incremental_join_groups", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_members", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_members_batch", g.GetIncrementalGroupMemberBatch) groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } @@ -192,6 +193,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/get_conversations", c.GetConversations) conversationGroup.POST("/set_conversations", c.SetConversations) conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) + conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) + conversationGroup.POST("/get_incremental_conversations", c.GetIncrementalConversation) + conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) } statisticsGroup := r.Group("/statistics") diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index df9267ae04..3047c376bb 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -184,13 +184,23 @@ func (c *conversationServer) GetAllConversations(ctx context.Context, req *pbcon } func (c *conversationServer) GetConversations(ctx context.Context, req *pbconversation.GetConversationsReq) (*pbconversation.GetConversationsResp, error) { - conversations, err := c.conversationDatabase.FindConversations(ctx, req.OwnerUserID, req.ConversationIDs) + conversations, err := c.getConversations(ctx, req.OwnerUserID, req.ConversationIDs) + if err != nil { + return nil, err + } + return &pbconversation.GetConversationsResp{ + Conversations: conversations, + }, nil +} + +func (c *conversationServer) getConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { + conversations, err := c.conversationDatabase.FindConversations(ctx, ownerUserID, conversationIDs) if err != nil { return nil, err } resp := &pbconversation.GetConversationsResp{Conversations: []*pbconversation.Conversation{}} resp.Conversations = convert.ConversationsDB2Pb(conversations) - return resp, nil + return convert.ConversationsDB2Pb(conversations), nil } func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { @@ -581,3 +591,14 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv } return &pbconversation.UpdateConversationResp{}, nil } + +func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbconversation.GetOwnerConversationReq) (*pbconversation.GetOwnerConversationResp, error) { + total, conversations, err := c.conversationDatabase.GetOwnerConversation(ctx, req.UserID, req.Pagination) + if err != nil { + return nil, err + } + return &pbconversation.GetOwnerConversationResp{ + Total: total, + Conversations: convert.ConversationsDB2Pb(conversations), + }, nil +} diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go new file mode 100644 index 0000000000..29c11c4a10 --- /dev/null +++ b/internal/rpc/conversation/sync.go @@ -0,0 +1,56 @@ +package conversation + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/conversation" +) + +func (c *conversationServer) GetFullOwnerConversationIDs(ctx context.Context, req *conversation.GetFullOwnerConversationIDsReq) (*conversation.GetFullOwnerConversationIDsResp, error) { + vl, err := c.conversationDatabase.FindMaxConversationUserVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(conversationIDs) + if req.IdHash == idHash { + conversationIDs = nil + } + return &conversation.GetFullOwnerConversationIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + ConversationIDs: conversationIDs, + }, nil +} + +func (c *conversationServer) GetIncrementalConversation(ctx context.Context, req *conversation.GetIncrementalConversationReq) (*conversation.GetIncrementalConversationResp, error) { + opt := incrversion.Option[*conversation.Conversation, conversation.GetIncrementalConversationResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: c.conversationDatabase.FindConversationUserVersion, + CacheMaxVersion: c.conversationDatabase.FindMaxConversationUserVersionCache, + Find: func(ctx context.Context, conversationIDs []string) ([]*conversation.Conversation, error) { + return c.getConversations(ctx, req.UserID, conversationIDs) + }, + ID: func(elem *conversation.Conversation) string { return elem.GroupID }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*conversation.Conversation, full bool) *conversation.GetIncrementalConversationResp { + return &conversation.GetIncrementalConversationResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 4cb1b81d06..881a5a3328 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -290,3 +290,8 @@ type FormDataMate struct { Group string `json:"group"` Key string `json:"key"` } + +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index b371f5e4e7..3ae5600330 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -44,6 +44,7 @@ type thirdServer struct { defaultExpire time.Duration config *Config } + type Config struct { RpcConfig config.Third RedisConfig config.Redis diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7b..144a37602b 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -736,3 +736,8 @@ func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (* } return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/pkg/common/storage/cache/cachekey/conversation.go b/pkg/common/storage/cache/cachekey/conversation.go index aea4ceec68..d19fcc5767 100644 --- a/pkg/common/storage/cache/cachekey/conversation.go +++ b/pkg/common/storage/cache/cachekey/conversation.go @@ -23,6 +23,7 @@ const ( SuperGroupRecvMsgNotNotifyUserIDsKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:" SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:" ConversationNotReceiveMessageUserIDsKey = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:" + ConversationUserMaxKey = "CONVERSATION_USER_MAX:" ) func GetConversationKey(ownerUserID, conversationID string) string { @@ -56,3 +57,7 @@ func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string { func GetUserConversationIDsHashKey(ownerUserID string) string { return ConversationIDsHashKey + ownerUserID } + +func GetConversationUserMaxVersionKey(userID string) string { + return ConversationUserMaxKey + userID +} diff --git a/pkg/common/storage/cache/conversation.go b/pkg/common/storage/cache/conversation.go index f34fd599f8..bc17614836 100644 --- a/pkg/common/storage/cache/conversation.go +++ b/pkg/common/storage/cache/conversation.go @@ -54,4 +54,8 @@ type ConversationCache interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache + + DelConversationVersionUserIDs(userIDs ...string) ConversationCache + + FindMaxConversationUserVersion(ctx context.Context, userID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 8c0393dd56..c491d1b947 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -95,6 +95,10 @@ func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID strin return cachekey.GetUserConversationIDsHashKey(ownerUserID) } +func (c *ConversationRedisCache) getConversationUserMaxVersionKey(ownerUserID string) string { + return cachekey.GetConversationUserMaxVersionKey(ownerUserID) +} + func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { return getCache(ctx, c.rcClient, c.getConversationIDsKey(ownerUserID), c.expireTime, func(ctx context.Context) ([]string, error) { return c.conversationDB.FindUserIDAllConversationID(ctx, ownerUserID) @@ -233,6 +237,19 @@ func (c *ConversationRedisCache) DelConversationNotReceiveMessageUserIDs(convers for _, conversationID := range conversationIDs { cache.AddKeys(c.getConversationNotReceiveMessageUserIDsKey(conversationID)) } + return cache +} +func (c *ConversationRedisCache) DelConversationVersionUserIDs(userIDs ...string) cache.ConversationCache { + cache := c.CloneConversationCache() + for _, userID := range userIDs { + cache.AddKeys(c.getConversationUserMaxVersionKey(userID)) + } return cache } + +func (c *ConversationRedisCache) FindMaxConversationUserVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, c.rcClient, c.getConversationUserMaxVersionKey(userID), c.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index 18ef3f8bad..c804d1cc51 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -66,6 +66,9 @@ type ConversationDatabase interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) // GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) // FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) + FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) + GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -106,6 +109,7 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, if _, ok := fieldMap["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) } + cache = cache.DelConversationVersionUserIDs(haveUserIDs...) } NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs) log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs) @@ -137,7 +141,7 @@ func (c *conversationDatabase) UpdateUsersConversationField(ctx context.Context, return err } cache := c.cache.CloneConversationCache() - cache = cache.DelUsersConversation(conversationID, userIDs...) + cache = cache.DelUsersConversation(conversationID, userIDs...).DelConversationVersionUserIDs(userIDs...) if _, ok := args["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversationID) } @@ -155,13 +159,14 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) userIDs = append(userIDs, conversation.OwnerUserID) } - return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ChainExecDel(ctx) + return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).DelConversationVersionUserIDs(userIDs...).ChainExecDel(ctx) } func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Context, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() for _, conversation := range conversations { + cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID) for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} { ownerUserID := v[0] userID := v[1] @@ -207,6 +212,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() + cache = cache.DelConversationVersionUserIDs(ownerUserID) groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.Conversation) (string, bool) { return e.GroupID, e.GroupID != "" })) @@ -322,3 +328,28 @@ func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Contex func (c *conversationDatabase) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { return c.cache.GetConversationNotReceiveMessageUserIDs(ctx, conversationID) } + +func (c *conversationDatabase) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, version, limit) +} + +func (c *conversationDatabase) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) { + return c.cache.FindMaxConversationUserVersion(ctx, userID) +} + +func (c *conversationDatabase) GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) { + conversationIDs, err := c.cache.GetUserConversationIDs(ctx, ownerUserID) + if err != nil { + return 0, nil, err + } + findConversationIDs := datautil.Paginate(conversationIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) + conversations := make([]*relationtb.Conversation, 0, len(findConversationIDs)) + for _, conversationID := range findConversationIDs { + conversation, err := c.cache.GetConversation(ctx, ownerUserID, conversationID) + if err != nil { + return 0, nil, err + } + conversations = append(conversations, conversation) + } + return int64(len(conversationIDs)), conversations, nil +} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 46aa02d98e..85f3dd668a 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -22,7 +22,6 @@ import ( type Conversation interface { Create(ctx context.Context, conversations []*model.Conversation) (err error) - Delete(ctx context.Context, groupIDs []string) (err error) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) Update(ctx context.Context, conversation *model.Conversation) (err error) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) @@ -39,4 +38,5 @@ type Conversation interface { GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index b462d39583..3d505f1d34 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -41,40 +41,71 @@ func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { if err != nil { return nil, errs.Wrap(err) } - return &ConversationMgo{coll: coll}, nil + version, err := NewVersionLog(db.Collection(database.ConversationVersionName)) + if err != nil { + return nil, err + } + return &ConversationMgo{version: version, coll: coll}, nil } type ConversationMgo struct { - coll *mongo.Collection + version database.VersionLog + coll *mongo.Collection } func (c *ConversationMgo) Create(ctx context.Context, conversations []*model.Conversation) (err error) { - return mongoutil.InsertMany(ctx, c.coll, conversations) -} - -func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err error) { - return mongoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, c.coll, conversations) + }, func() error { + userConversation := make(map[string][]string) + for _, conversation := range conversations { + userConversation[conversation.OwnerUserID] = append(userConversation[conversation.OwnerUserID], conversation.ConversationID) + } + for userID, conversationIDs := range userConversation { + if err := c.version.IncrVersion(ctx, userID, conversationIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } -func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) { - if len(args) == 0 { +func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (int64, error) { + if len(args) == 0 || len(userIDs) == 0 { return 0, nil } filter := bson.M{ "conversation_id": conversationID, + "owner_user_id": bson.M{"$in": userIDs}, } - if len(userIDs) > 0 { - filter["owner_user_id"] = bson.M{"$in": userIDs} - } - res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + var rows int64 + err := mongoutil.IncrVersion(func() error { + res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + if err != nil { + return err + } + rows = res.ModifiedCount + return nil + }, func() error { + for _, userID := range userIDs { + if err := c.version.IncrVersion(ctx, userID, []string{conversationID}, model.VersionStateUpdate); err != nil { + return err + } + } + return nil + }) if err != nil { return 0, err } - return res.ModifiedCount, nil + return rows, nil } func (c *ConversationMgo) Update(ctx context.Context, conversation *model.Conversation) (err error) { - return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + }, func() error { + return c.version.IncrVersion(ctx, conversation.OwnerUserID, []string{conversation.ConversationID}, model.VersionStateUpdate) + }) } func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) { @@ -178,3 +209,7 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}), ) } + +func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return c.version.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 04aa59cef9..ebabadf52c 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -205,7 +205,7 @@ func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID strin if sort != 1 && sort != -1 { return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1") } - opt := options.Find().SetLimit(1).SetSkip(index).SetSort(bson.M{"doc_id": sort}).SetLimit(1) + opt := options.Find().SetSkip(index).SetSort(bson.M{"_id": sort}).SetLimit(1) filter := bson.M{"doc_id": primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)}} msgs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, filter, opt) if err != nil { diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 986f22a1a2..3d9eb6a977 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -1,17 +1,18 @@ package database const ( - BlackName = "black" - ConversationName = "conversation" - FriendName = "friend" - FriendVersionName = "friend_version" - FriendRequestName = "friend_request" - GroupName = "group" - GroupMemberName = "group_member" - GroupMemberVersionName = "group_member_version" - GroupJoinVersionName = "group_join_version" - GroupRequestName = "group_request" - LogName = "log" - ObjectName = "s3" - UserName = "user" + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + ConversationVersionName = "conversation_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" ) From 4aaf496086d72aa251553b76882831a503cb8757 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:46:21 +0800 Subject: [PATCH 16/91] feat: new features merged (#2409) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- .env | 3 + .gitignore | 4 - cmd/openim-api/main.go | 1 - config/alertmanager.yml | 25 + config/email.tmpl | 16 + config/instance-down-rules.yml | 22 + config/openim-crontask.yml | 3 +- config/prometheus.yml | 83 ++ docker-compose.yml | 45 + go.mod | 4 +- go.sum | 4 +- internal/api/friend.go | 1 - internal/api/init.go | 7 +- internal/api/router.go | 23 +- internal/msggateway/client.go | 4 + internal/msggateway/compressor_test.go | 9 +- internal/msggateway/constant.go | 1 + internal/msggateway/hub_server.go | 10 +- internal/msggateway/init.go | 20 +- internal/msggateway/online.go | 112 +++ internal/msggateway/subscription.go | 181 ++++ internal/msggateway/user_map.go | 244 +++-- internal/msggateway/ws_server.go | 58 +- internal/msgtransfer/init.go | 28 +- internal/push/push_handler.go | 45 +- internal/rpc/msg/server.go | 13 +- internal/rpc/third/s3.go | 57 +- internal/rpc/third/third.go | 16 +- internal/rpc/user/online.go | 122 +++ internal/rpc/user/user.go | 78 +- internal/tools/cron_task.go | 28 +- pkg/apistruct/manage.go | 2 +- pkg/common/cmd/msg_gateway.go | 1 + pkg/common/config/config.go | 5 +- pkg/common/ginprometheus/ginprometheus.go | 855 +++++++++--------- pkg/common/prommetrics/api.go | 48 + pkg/common/prommetrics/gin_api.go | 30 - pkg/common/prommetrics/prommetrics.go | 46 +- pkg/common/prommetrics/prommetrics_test.go | 73 +- pkg/common/prommetrics/rpc.go | 58 ++ pkg/common/prommetrics/transfer.go | 14 + pkg/common/startrpc/start.go | 64 +- pkg/common/storage/cache/cachekey/online.go | 13 + pkg/common/storage/cache/cachekey/seq.go | 44 +- pkg/common/storage/cache/cachekey/user.go | 5 - pkg/common/storage/cache/group.go | 1 - pkg/common/storage/cache/online.go | 8 + pkg/common/storage/cache/redis/batch.go | 94 ++ .../storage/cache/redis/batch_handler.go | 55 +- pkg/common/storage/cache/redis/batch_test.go | 55 ++ .../storage/cache/redis/conversation.go | 8 +- pkg/common/storage/cache/redis/friend.go | 25 - pkg/common/storage/cache/redis/group.go | 90 +- pkg/common/storage/cache/redis/msg.go | 1 - pkg/common/storage/cache/redis/online.go | 89 ++ .../cache/redis/redis_shard_manager.go | 16 +- pkg/common/storage/cache/redis/seq.go | 200 ---- .../storage/cache/redis/seq_conversation.go | 333 +++++++ .../cache/redis/seq_conversation_test.go | 109 +++ pkg/common/storage/cache/redis/seq_user.go | 185 ++++ .../storage/cache/redis/seq_user_test.go | 79 ++ pkg/common/storage/cache/redis/user.go | 188 +--- pkg/common/storage/cache/seq.go | 30 - pkg/common/storage/cache/seq_conversation.go | 12 + pkg/common/storage/cache/seq_user.go | 15 + pkg/common/storage/cache/user.go | 5 +- pkg/common/storage/controller/msg.go | 197 ++-- pkg/common/storage/controller/s3.go | 46 +- pkg/common/storage/controller/third.go | 3 +- pkg/common/storage/controller/user.go | 17 +- pkg/common/storage/database/group_member.go | 2 + .../storage/database/mgo/group_member.go | 16 + pkg/common/storage/database/mgo/object.go | 14 + .../storage/database/mgo/seq_conversation.go | 103 +++ .../database/mgo/seq_conversation_test.go | 37 + pkg/common/storage/database/mgo/seq_user.go | 110 +++ pkg/common/storage/database/name.go | 2 + pkg/common/storage/database/object.go | 5 + pkg/common/storage/database/seq.go | 11 + pkg/common/storage/database/seq_user.go | 13 + pkg/common/storage/model/seq.go | 7 + pkg/common/storage/model/seq_user.go | 9 + pkg/localcache/cache.go | 12 +- pkg/localcache/lru/lru.go | 1 + pkg/localcache/lru/lru_expiration.go | 10 + pkg/localcache/lru/lru_lazy.go | 22 + pkg/localcache/lru/lru_slot.go | 4 + pkg/localcache/option.go | 14 +- pkg/msgprocessor/conversation.go | 4 + pkg/rpccache/online.go | 100 ++ pkg/rpccache/user.go | 15 + pkg/rpcclient/third.go | 4 + pkg/rpcclient/user.go | 22 + pkg/util/useronline/split.go | 27 + start-config.yml | 1 + tools/seq/internal/main.go | 331 +++++++ tools/seq/main.go | 22 + 97 files changed, 3730 insertions(+), 1574 deletions(-) create mode 100644 config/alertmanager.yml create mode 100644 config/email.tmpl create mode 100644 config/instance-down-rules.yml create mode 100644 config/prometheus.yml create mode 100644 internal/msggateway/online.go create mode 100644 internal/msggateway/subscription.go create mode 100644 internal/rpc/user/online.go create mode 100644 pkg/common/prommetrics/api.go delete mode 100644 pkg/common/prommetrics/gin_api.go create mode 100644 pkg/common/prommetrics/rpc.go create mode 100644 pkg/common/storage/cache/cachekey/online.go create mode 100644 pkg/common/storage/cache/online.go create mode 100644 pkg/common/storage/cache/redis/batch.go create mode 100644 pkg/common/storage/cache/redis/batch_test.go create mode 100644 pkg/common/storage/cache/redis/online.go delete mode 100644 pkg/common/storage/cache/redis/seq.go create mode 100644 pkg/common/storage/cache/redis/seq_conversation.go create mode 100644 pkg/common/storage/cache/redis/seq_conversation_test.go create mode 100644 pkg/common/storage/cache/redis/seq_user.go create mode 100644 pkg/common/storage/cache/redis/seq_user_test.go delete mode 100644 pkg/common/storage/cache/seq.go create mode 100644 pkg/common/storage/cache/seq_conversation.go create mode 100644 pkg/common/storage/cache/seq_user.go create mode 100644 pkg/common/storage/database/mgo/seq_conversation.go create mode 100644 pkg/common/storage/database/mgo/seq_conversation_test.go create mode 100644 pkg/common/storage/database/mgo/seq_user.go create mode 100644 pkg/common/storage/database/seq.go create mode 100644 pkg/common/storage/database/seq_user.go create mode 100644 pkg/common/storage/model/seq.go create mode 100644 pkg/common/storage/model/seq_user.go create mode 100644 pkg/rpccache/online.go create mode 100644 pkg/util/useronline/split.go create mode 100644 tools/seq/internal/main.go create mode 100644 tools/seq/main.go diff --git a/.env b/.env index 1e7b1e11a7..3199b37149 100644 --- a/.env +++ b/.env @@ -5,6 +5,9 @@ ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 +PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 +ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 +GRAFANA_IMAGE=grafana/grafana:11.0.1 OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.5.1 OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.7 diff --git a/.gitignore b/.gitignore index fb8d428d24..77cf855b70 100644 --- a/.gitignore +++ b/.gitignore @@ -34,11 +34,7 @@ deployments/charts/generated-configs/ ### OpenIM Config ### .env config/config.yaml -config/alertmanager.yml -config/prometheus.yml -config/email.tmpl config/notification.yaml -config/instance-down-rules.yml ### OpenIM deploy ### deployments/openim-server/charts diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index 58e540c05f..e29ed2a592 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -25,5 +25,4 @@ func main() { if err := cmd.NewApiCmd().Exec(); err != nil { program.ExitWithError(err) } - } diff --git a/config/alertmanager.yml b/config/alertmanager.yml new file mode 100644 index 0000000000..a029448518 --- /dev/null +++ b/config/alertmanager.yml @@ -0,0 +1,25 @@ +global: + resolve_timeout: 5m + smtp_from: alert@openim.io + smtp_smarthost: smtp.163.com:465 + smtp_auth_username: alert@openim.io + smtp_auth_password: YOURAUTHPASSWORD + smtp_require_tls: false + smtp_hello: xxx + +templates: + - /etc/alertmanager/email.tmpl + +route: + group_by: ['alertname'] + group_wait: 5s + group_interval: 5s + repeat_interval: 5m + receiver: email +receivers: + - name: email + email_configs: + - to: 'alert@example.com' + html: '{{ template "email.to.html" . }}' + headers: { Subject: "[OPENIM-SERVER]Alarm" } + send_resolved: true diff --git a/config/email.tmpl b/config/email.tmpl new file mode 100644 index 0000000000..0385601d00 --- /dev/null +++ b/config/email.tmpl @@ -0,0 +1,16 @@ +{{ define "email.to.html" }} +{{ range .Alerts }} + +
+

OpenIM Alert

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ +{{ end }} +{{ end }} diff --git a/config/instance-down-rules.yml b/config/instance-down-rules.yml new file mode 100644 index 0000000000..5541d2c542 --- /dev/null +++ b/config/instance-down-rules.yml @@ -0,0 +1,22 @@ +groups: + - name: instance_down + rules: + - alert: InstanceDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes." + + - name: database_insert_failure_alerts + rules: + - alert: DatabaseInsertFailed + expr: (increase(msg_insert_redis_failed_total[5m]) > 0) or (increase(msg_insert_mongo_failed_total[5m]) > 0) + for: 1m + labels: + severity: critical + annotations: + summary: "Increase in MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter detected" + description: "Either MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter has increased in the last 5 minutes, indicating failures in message insert operations to Redis or MongoDB,maybe the redis or mongodb is crash." diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 9bbccfd252..3839104a44 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,2 +1,3 @@ -chatRecordsClearTime: "0 2 * * *" +cronExecuteTime: "0 2 * * *" retainChatRecords: 365 +fileExpireTime: 90 diff --git a/config/prometheus.yml b/config/prometheus.yml new file mode 100644 index 0000000000..99f2e4df34 --- /dev/null +++ b/config/prometheus.yml @@ -0,0 +1,83 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: ['192.168.2.22:19093'] + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "instance-down-rules.yml" +# - "first_rules.yml" +# - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label "job='job_name'"" to any timeseries scraped from this config. + # Monitored information captured by prometheus + + # prometheus fetches application services + - job_name: 'node_exporter' + static_configs: + - targets: [ '192.168.2.22:20114' ] + - job_name: 'openimserver-openim-api' + static_configs: + - targets: [ '192.168.2.22:20113' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-msggateway' + static_configs: + - targets: [ '192.168.2.22:20112' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-msgtransfer' + static_configs: + - targets: [ 192.168.2.22:20111, 192.168.2.22:20110, 192.168.2.22:20109, 192.168.2.22:20108 ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-push' + static_configs: + - targets: [ '192.168.2.22:20107' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-auth' + static_configs: + - targets: [ '192.168.2.22:20106' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-conversation' + static_configs: + - targets: [ '192.168.2.22:20105' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-friend' + static_configs: + - targets: [ '192.168.2.22:20104' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-group' + static_configs: + - targets: [ '192.168.2.22:20103' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-msg' + static_configs: + - targets: [ '192.168.2.22:20102' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-third' + static_configs: + - targets: [ '192.168.2.22:20101' ] + labels: + namespace: 'default' + - job_name: 'openimserver-openim-rpc-user' + static_configs: + - targets: [ '192.168.2.22:20100' ] + labels: + namespace: 'default' \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d72c1a2fa4..8cc1f24b2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,5 +140,50 @@ services: networks: - openim + prometheus: + image: ${PROMETHEUS_IMAGE} + container_name: prometheus + restart: always + volumes: + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml + - ${DATA_DIR}/components/prometheus/data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + ports: + - "19091:9090" + networks: + - openim + + alertmanager: + image: ${ALERTMANAGER_IMAGE} + container_name: alertmanager + restart: always + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + - ./config/email.tmpl:/etc/alertmanager/email.tmpl + ports: + - "19093:9093" + networks: + - openim + + grafana: + image: ${GRAFANA_IMAGE} + container_name: grafana + user: root + restart: always + environment: + - GF_SECURITY_ALLOW_EMBEDDING=true + - GF_SESSION_COOKIE_SAMESITE=none + - GF_SESSION_COOKIE_SECURE=true + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "13000:3000" + volumes: + - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana + networks: + - openim diff --git a/go.mod b/go.mod index 207c2cf5e7..2e47d1b5e5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.30 - github.com/openimsdk/tools v0.0.49-alpha.49 + github.com/openimsdk/tools v0.0.49-alpha.50 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -194,5 +194,3 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index ef8f68a912..649150dec7 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.49 h1:bkVxlEDM9rJxo5a98mk2ETR0xbEv+5mjeVVVFQfARAQ= -github.com/openimsdk/tools v0.0.49-alpha.49/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/tools v0.0.49-alpha.50 h1:7CaYLVtsBU5kyiTetUOuOkO5FFFmMvSzBEfh2tfCn90= +github.com/openimsdk/tools v0.0.49-alpha.50/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/friend.go b/internal/api/friend.go index 6c42184230..f9f15fb246 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -58,7 +58,6 @@ func (o *FriendApi) GetFriendList(c *gin.Context) { func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) - //a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c, a2r.NewNilReplaceOption(relation.FriendClient.GetDesignatedFriends)) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { diff --git a/internal/api/init.go b/internal/api/init.go index 23866c4a07..e83dfc2ea5 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,7 +29,6 @@ import ( "time" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" @@ -72,10 +71,8 @@ func Start(ctx context.Context, index int, config *Config) error { netDone <- struct{}{} return } - p := ginprom.NewPrometheus("app", prommetrics.GetGinCusMetrics("Api")) - p.SetListenAddress(fmt.Sprintf(":%d", prometheusPort)) - if err = p.Use(router); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) + if err := prommetrics.ApiInit(prometheusPort); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } }() diff --git a/internal/api/router.go b/internal/api/router.go index 822e647932..0667c3e751 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,15 +2,13 @@ package api import ( "fmt" - "net/http" - "strings" - "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -18,8 +16,25 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" + "net/http" + "strings" ) +func prommetricsGin() gin.HandlerFunc { + return func(c *gin.Context) { + c.Next() + path := c.FullPath() + if c.Writer.Status() == http.StatusNotFound { + prommetrics.HttpCall("<404>", c.Request.Method, c.Writer.Status()) + } else { + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + } + if resp := apiresp.GetGinApiResponse(c); resp != nil { + prommetrics.APICall(path, c.Request.Method, resp.ErrCode) + } + } +} + func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) @@ -38,7 +53,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - r.Use(gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) userRouterGroup := r.Group("/user") diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 1270eb9781..446553dc08 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -75,6 +75,8 @@ type Client struct { token string hbCtx context.Context hbCancel context.CancelFunc + subLock sync.Mutex + subUserIDs map[string]struct{} } // ResetClient updates the client's state with new connection and context information. @@ -216,6 +218,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) + case WsSubUserOnlineStatus: + resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/compressor_test.go b/internal/msggateway/compressor_test.go index 173c9bb204..952bd4d95b 100644 --- a/internal/msggateway/compressor_test.go +++ b/internal/msggateway/compressor_test.go @@ -16,10 +16,10 @@ package msggateway import ( "crypto/rand" + "github.com/stretchr/testify/assert" "sync" "testing" - - "github.com/stretchr/testify/assert" + "unsafe" ) func mockRandom() []byte { @@ -132,3 +132,8 @@ func BenchmarkDecompressWithSyncPool(b *testing.B) { assert.Equal(b, nil, err) } } + +func TestName(t *testing.T) { + t.Log(unsafe.Sizeof(Client{})) + +} diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 125be16358..dc5ad77861 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -43,6 +43,7 @@ const ( WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 + WsSubUserOnlineStatus = 2005 WSDataError = 3001 ) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8ff6d10018..3891aa532c 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/tools/discovery" @@ -31,6 +32,10 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + if s.ready != nil { + return s.ready(s) + } return nil } @@ -50,18 +55,21 @@ type Server struct { LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} + ready func(srv *Server) error + userRcp rpcclient.UserRpcClient } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config) *Server { +func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, + ready: ready, } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index f4d8b0381a..44e79e4122 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,8 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" + "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -26,6 +28,7 @@ import ( type Config struct { MsgGateway config.MsgGateway Share config.Share + RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery } @@ -42,18 +45,25 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } - longServer, err := NewWsServer( + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) + if err != nil { + return err + } + longServer := NewWsServer( conf, WithPort(wsPort), WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)), WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second), WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - if err != nil { - return err - } - hubServer := NewServer(rpcPort, longServer, conf) + hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + longServer.online = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, longServer.subscriberUserOnlineStatusChanges) + return nil + }) + + go longServer.ChangeOnlineStatus(4) + netDone := make(chan error) go func() { err = hubServer.Start(ctx, index, conf) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go new file mode 100644 index 0000000000..b50608f93c --- /dev/null +++ b/internal/msggateway/online.go @@ -0,0 +1,112 @@ +package msggateway + +import ( + "context" + "crypto/md5" + "encoding/binary" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" + "math/rand" + "strconv" + "time" +) + +func (ws *WsServer) ChangeOnlineStatus(concurrent int) { + if concurrent < 1 { + concurrent = 1 + } + const renewalTime = cachekey.OnlineExpire / 3 + //const renewalTime = time.Second * 10 + renewalTicker := time.NewTicker(renewalTime) + + requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) + changeStatus := make([][]UserState, concurrent) + + for i := 0; i < concurrent; i++ { + requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) + changeStatus[i] = make([]UserState, 0, 100) + } + + mergeTicker := time.NewTicker(time.Second) + + local2pb := func(u UserState) *pbuser.UserOnlineStatus { + return &pbuser.UserOnlineStatus{ + UserID: u.UserID, + Online: u.Online, + Offline: u.Offline, + } + } + + rNum := rand.Uint64() + pushUserState := func(us ...UserState) { + for _, u := range us { + sum := md5.Sum([]byte(u.UserID)) + i := (binary.BigEndian.Uint64(sum[:]) + rNum) % uint64(concurrent) + changeStatus[i] = append(changeStatus[i], u) + status := changeStatus[i] + if len(status) == cap(status) { + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[:0] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + } + + pushAllUserState := func() { + for i, status := range changeStatus { + if len(status) == 0 { + continue + } + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[:0] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + + opIdCtx := mcontext.SetOperationID(context.Background(), "r"+strconv.FormatUint(rNum, 10)) + doRequest := func(req *pbuser.SetUserOnlineStatusReq) { + ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) + defer cancel() + if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { + log.ZError(ctx, "update user online status", err) + } + } + + for i := 0; i < concurrent; i++ { + go func(ch <-chan *pbuser.SetUserOnlineStatusReq) { + for req := range ch { + doRequest(req) + } + }(requestChs[i]) + } + + for { + select { + case <-mergeTicker.C: + pushAllUserState() + case now := <-renewalTicker.C: + deadline := now.Add(-cachekey.OnlineExpire / 3) + users := ws.clients.GetAllUserStatus(deadline, now) + log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users)) + pushUserState(users...) + case state := <-ws.clients.UserState(): + log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) + pushUserState(state) + } + } +} diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go new file mode 100644 index 0000000000..9460f5dbfb --- /dev/null +++ b/internal/msggateway/subscription.go @@ -0,0 +1,181 @@ +package msggateway + +import ( + "context" + "encoding/json" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/idutil" + "google.golang.org/protobuf/proto" + "sync" + "time" +) + +func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + } + ws.pushUserIDOnlineStatus(ctx, userID, platformIDs) +} + +func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) { + var sub sdkws.SubUserOnlineStatus + if err := proto.Unmarshal(data.Data, &sub); err != nil { + return nil, err + } + ws.subscription.Sub(client, sub.SubscribeUserID, sub.UnsubscribeUserID) + var resp sdkws.SubUserOnlineStatusTips + if len(sub.SubscribeUserID) > 0 { + resp.Subscribers = make([]*sdkws.SubUserOnlineStatusElem, 0, len(sub.SubscribeUserID)) + for _, userID := range sub.SubscribeUserID { + platformIDs, err := ws.online.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + resp.Subscribers = append(resp.Subscribers, &sdkws.SubUserOnlineStatusElem{ + UserID: userID, + OnlinePlatformIDs: platformIDs, + }) + } + } + return proto.Marshal(&resp) +} + +type subClient struct { + clients map[string]*Client +} + +func newSubscription() *Subscription { + return &Subscription{ + userIDs: make(map[string]*subClient), + } +} + +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient +} + +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + +func (s *Subscription) DelClient(client *Client) { + client.subLock.Lock() + userIDs := datautil.Keys(client.subUserIDs) + for _, userID := range userIDs { + delete(client.subUserIDs, userID) + } + client.subLock.Unlock() + if len(userIDs) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for _, userID := range userIDs { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } +} + +func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { + if len(addUserIDs)+len(delUserIDs) == 0 { + return + } + var ( + del = make(map[string]struct{}) + add = make(map[string]struct{}) + ) + client.subLock.Lock() + for _, userID := range delUserIDs { + if _, ok := client.subUserIDs[userID]; !ok { + continue + } + del[userID] = struct{}{} + delete(client.subUserIDs, userID) + } + for _, userID := range addUserIDs { + delete(del, userID) + if _, ok := client.subUserIDs[userID]; ok { + continue + } + client.subUserIDs[userID] = struct{}{} + } + client.subLock.Unlock() + if len(del)+len(add) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for userID := range del { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } + for userID := range add { + sub, ok := s.userIDs[userID] + if !ok { + sub = &subClient{clients: make(map[string]*Client)} + s.userIDs[userID] = sub + } + sub.clients[addr] = client + } +} + +func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { + clients := ws.subscription.GetClient(userID) + if len(clients) == 0 { + return + } + msgContent, err := json.Marshal(platformIDs) + if err != nil { + log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) + return + } + now := time.Now().UnixMilli() + msgID := idutil.GetMsgIDByMD5(userID) + msg := &sdkws.MsgData{ + SendID: userID, + ClientMsgID: msgID, + ServerMsgID: msgID, + SenderPlatformID: constant.AdminPlatformID, + SessionType: constant.NotificationChatType, + ContentType: constant.UserSubscribeOnlineStatusNotification, + Content: msgContent, + SendTime: now, + CreateTime: now, + } + for _, client := range clients { + msg.RecvID = client.UserID + if err := client.PushMessage(ctx, msg); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) + } + } +} diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 79cc53d1bc..bd1f197281 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,135 +1,185 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package msggateway import ( - "context" - "sync" - - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" + "sync" + "time" ) -type UserMap struct { - m sync.Map +type UserMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState + RecvSubChange(userID string, platformIDs []int32) bool } -func newUserMap() *UserMap { - return &UserMap{} +type UserState struct { + UserID string + Online []int32 + Offline []int32 } -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.([]*Client), ok - } - return nil, ok +type UserPlatform struct { + Time time.Time + Clients []*Client } -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.([]*Client) { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, userExisted, true - } - return clients, userExisted, false +func (u *UserPlatform) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) } - return nil, userExisted, false + return platformIDs } -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.([]*Client) - oldClients = append(oldClients, v) - u.m.Store(key, oldClients) - } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) +func (u *UserPlatform) PlatformIDSet() map[int32]struct{} { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make(map[int32]struct{}) + for _, client := range u.Clients { + platformIDs[int32(client.PlatformID)] = struct{}{} + } + return platformIDs +} - var clients []*Client - clients = append(clients, v) - u.m.Store(key, clients) +func newUserMap() UserMap { + return &userMap{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), } } -func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) { - // Attempt to load the clients associated with the key. - allClients, existed := u.m.Load(key) - if !existed { - // Return false immediately if the key does not exist. +type userMap struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState +} + +func (u *userMap) RecvSubChange(userID string, platformIDs []int32) bool { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { return false } - - // Convert allClients to a slice of *Client. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - // Keep clients that do not match the connRemoteAddr. - if client.ctx.GetRemoteAddr() != connRemoteAddr { - remainingClients = append(remainingClients, client) - } + localPlatformIDs := result.PlatformIDSet() + for _, platformID := range platformIDs { + delete(localPlatformIDs, platformID) } + if len(localPlatformIDs) == 0 { + return false + } + u.push(userID, result, nil) + return true +} - // If no clients remain after filtering, delete the key from the map. - if len(remainingClients) == 0 { - u.m.Delete(key) +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() return true + default: + return false } +} - // Otherwise, update the key with the remaining clients. - u.m.Store(key, remainingClients) - return false +func (u *userMap) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false + } + return result.Clients, true } -func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. - return false +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + return clients, true, len(clients) > 0 +} - // Filter out clients that are in the deleteMap. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) +func (u *userMap) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients = append(result.Clients, client) + } else { + result = &UserPlatform{ + Clients: []*Client{client}, } + u.data[userID] = result } + u.push(client.UserID, result, nil) +} - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { + return false } + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, delCli := deleteAddr[client.ctx.GetRemoteAddr()]; delCli { + offline = append(offline, int32(client.PlatformID)) + } else { + result.Clients = append(result.Clients, client) + } + } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} - u.m.Store(key, remainingClients) - return false +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { + u.lock.RLock() + defer u.lock.RUnlock() + result := make([]UserState, 0, len(u.data)) + for userID, userPlatform := range u.data { + if userPlatform.Time.Before(deadline) { + continue + } + userPlatform.Time = nowtime + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) + } + return result } -func (u *UserMap) DeleteAll(key string) { - u.m.Delete(key) +func (u *userMap) UserState() <-chan UserState { + return u.ch } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index defec16df1..e903084a9d 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/mcontext" "net/http" @@ -48,6 +49,7 @@ type LongConnServer interface { KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) + SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor Encoder MessageHandler @@ -60,7 +62,9 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients *UserMap + clients UserMap + online *rpccache.OnlineCache + subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -90,18 +94,18 @@ func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, ws.disCov = disCov } -func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { - err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) - if err != nil { - log.ZWarn(ctx, "SetUserStatus err", err) - } - switch status { - case constant.Online: - ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) - case constant.Offline: - ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) - } -} +//func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { +// err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) +// if err != nil { +// log.ZWarn(ctx, "SetUserStatus err", err) +// } +// switch status { +// case constant.Online: +// ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) +// case constant.Offline: +// ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) +// } +//} func (ws *WsServer) UnRegister(c *Client) { ws.unregisterChan <- c @@ -119,11 +123,13 @@ func (ws *WsServer) GetUserPlatformCons(userID string, platform int) ([]*Client, return ws.clients.Get(userID, platform) } -func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { +func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { var config configs for _, o := range opts { o(&config) } + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + v := validator.New() return &WsServer{ msgGatewayConfig: msgGatewayConfig, @@ -141,10 +147,11 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), + subscription: newSubscription(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), - }, nil + } } func (ws *WsServer) Run(done chan error) error { @@ -278,11 +285,11 @@ func (ws *WsServer) registerClient(client *Client) { }() } - wg.Add(1) - go func() { - defer wg.Done() - ws.SetUserOnlineStatus(client.ctx, client, constant.Online) - }() + //wg.Add(1) + //go func() { + // defer wg.Done() + // ws.SetUserOnlineStatus(client.ctx, client, constant.Online) + //}() wg.Wait() @@ -309,7 +316,7 @@ func getRemoteAdders(client []*Client) string { } func (ws *WsServer) KickUserConn(client *Client) error { - ws.clients.deleteClients(client.UserID, []*Client{client}) + ws.clients.DeleteClients(client.UserID, []*Client{client}) return client.KickOnlineMessage() } @@ -325,7 +332,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if !clientOK { return } - ws.clients.deleteClients(newClient.UserID, oldClients) + ws.clients.DeleteClients(newClient.UserID, oldClients) for _, c := range oldClients { err := c.KickOnlineMessage() if err != nil { @@ -345,13 +352,16 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.DeleteClients(client.UserID, []*Client{client}) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) + client.subLock.Lock() + clear(client.subUserIDs) + client.subLock.Unlock() + //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 65d04f3810..b4b2245eb0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -17,6 +17,7 @@ package msgtransfer import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" @@ -29,16 +30,12 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -82,12 +79,21 @@ func Start(ctx context.Context, index int, config *Config) error { client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err } - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } @@ -130,14 +136,8 @@ func (m *MsgTransfer) Start(index int, config *Config) error { netDone <- struct{}{} return } - proreg := prometheus.NewRegistry() - proreg.MustRegister( - collectors.NewGoCollector(), - ) - proreg.MustRegister(prommetrics.GetGrpcCusMetrics("Transfer", &config.Share)...) - http.Handle("/metrics", promhttp.HandlerFor(proreg, promhttp.HandlerOpts{Registry: proreg})) - err = http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), nil) - if err != nil && err != http.ErrServerClosed { + + if err := prommetrics.TransferInit(prometheusPort); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index dfe0e7b55d..ed87b3929f 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -28,6 +28,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" pbchat "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -45,6 +46,7 @@ type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache msgRpcClient rpcclient.MessageRpcClient @@ -63,16 +65,17 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, - &config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config + consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) return &consumerHandler, nil } @@ -125,12 +128,12 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. -func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) error { +func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, userIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } @@ -179,6 +182,38 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat return true } +func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { + var ( + onlineUserIDs []string + offlineUserIDs []string + ) + for _, userID := range pushToUserIDs { + online, err := c.onlineCache.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + var result []*msggateway.SingleMsgToUserResults + if len(onlineUserIDs) > 0 { + var err error + result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + if err != nil { + return nil, err + } + } + for _, userID := range offlineUserIDs { + result = append(result, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return result, nil +} + func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { log.ZDebug(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) var pushToUserIDs []string @@ -192,7 +227,7 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index f1fb28ffff..de0f698ea3 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -86,12 +86,21 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 881a5a3328..f96eb73905 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,13 +19,17 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -283,6 +287,52 @@ func (t *thirdServer) apiAddress(prefix, name string) string { return prefix + name } +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + var conf config.Third + expireTime := time.UnixMilli(req.ExpireTime) + findPagination := &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 1000, + } + for { + total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 && t.minio != nil { + thumbnailKey, err := t.getMinioImageThumbnailKey(ctx, key) + if err != nil { + return nil, errs.Wrap(err) + } + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) + t.s3dataBase.DeleteObject(ctx, key) + } + } + for _, model := range models { + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + if err != nil { + return nil, errs.Wrap(err) + } + } + if total < int64(findPagination.ShowNumber) { + break + } + } + return &third.DeleteOutdatedDataResp{}, nil +} + type FormDataMate struct { Name string `json:"name"` Size int64 `json:"size"` @@ -290,8 +340,3 @@ type FormDataMate struct { Group string `json:"group"` Key string `json:"key"` } - -func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - //TODO implement me - panic("implement me") -} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 3ae5600330..0eeaaa3140 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -31,9 +31,9 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "github.com/openimsdk/tools/s3/kodo" "google.golang.org/grpc" ) @@ -43,6 +43,7 @@ type thirdServer struct { userRpcClient rpcclient.UserRpcClient defaultExpire time.Duration config *Config + minio *minio.Minio } type Config struct { @@ -75,10 +76,14 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable - var o s3.Interface + var ( + o s3.Interface + minioCli *minio.Minio + ) switch enable { case "minio": - o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + o = minioCli case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -98,10 +103,15 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, + minio: minioCli, }) return nil } +func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { + return t.minio.GetImageThumbnailKey(ctx, name) +} + func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go new file mode 100644 index 0000000000..e853ceae23 --- /dev/null +++ b/internal/rpc/user/online.go @@ -0,0 +1,122 @@ +package user + +import ( + "context" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" + pbuser "github.com/openimsdk/protocol/user" +) + +func (s *userServer) getUserOnlineStatus(ctx context.Context, userID string) (*pbuser.OnlineStatus, error) { + platformIDs, err := s.online.GetOnline(ctx, userID) + if err != nil { + return nil, err + } + status := pbuser.OnlineStatus{ + UserID: userID, + PlatformIDs: platformIDs, + } + if len(platformIDs) > 0 { + status.Status = constant.Online + } else { + status.Status = constant.Offline + } + return &status, nil +} + +func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) ([]*pbuser.OnlineStatus, error) { + res := make([]*pbuser.OnlineStatus, 0, len(userIDs)) + for _, userID := range userIDs { + status, err := s.getUserOnlineStatus(ctx, userID) + if err != nil { + return nil, err + } + res = append(res, status) + } + return res, nil +} + +// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. +func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { + if req.Genre == constant.SubscriberUser { + err := s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + var status []*pbuser.OnlineStatus + status, err = s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil + } else if req.Genre == constant.Unsubscribe { + err := s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + } + return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil +} + +// GetUserStatus Get the online status of the user. +func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (*pbuser.GetUserStatusResp, error) { + res, err := s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.GetUserStatusResp{StatusList: res}, nil +} + +// SetUserStatus Synchronize user's online status. +func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (*pbuser.SetUserStatusResp, error) { + var ( + online []int32 + offline []int32 + ) + switch req.Status { + case constant.Online: + online = []int32{req.PlatformID} + case constant.Offline: + online = []int32{req.PlatformID} + } + if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { + return nil, err + } + list, err := s.db.GetSubscribedList(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range list { + tips := &sdkws.UserStatusChangeTips{ + FromUserID: req.UserID, + ToUserID: userID, + Status: req.Status, + PlatformID: req.PlatformID, + } + s.userNotificationSender.UserStatusChangeNotification(ctx, tips) + } + + return &pbuser.SetUserStatusResp{}, nil +} + +// GetSubscribeUsersStatus Get the online status of subscribers. +func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) + if err != nil { + return nil, err + } + onlineStatusList, err := s.getUsersOnlineStatus(ctx, userList) + if err != nil { + return nil, err + } + return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil +} + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + for _, status := range req.Status { + if err := s.online.SetUserOnline(ctx, status.UserID, status.Online, status.Offline); err != nil { + return nil, err + } + } + return &pbuser.SetUserOnlineStatusResp{}, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 144a37602b..0b96077ec3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -19,6 +19,7 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -50,6 +51,7 @@ import ( ) type userServer struct { + online cache.OnlineCache db controller.UserDatabase friendNotificationSender *friend.FriendNotificationSender userNotificationSender *UserNotificationSender @@ -98,6 +100,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ + online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, friendRpcClient: &friendRpcClient, @@ -329,76 +332,6 @@ func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDR return &pbuser.GetAllUserIDResp{Total: int32(total), UserIDs: userIDs}, nil } -// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. -func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (resp *pbuser.SubscribeOrCancelUsersStatusResp, err error) { - if req.Genre == constant.SubscriberUser { - err = s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err = s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } - return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil -} - -// GetUserStatus Get the online status of the user. -func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { - onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.GetUserStatusResp{StatusList: onlineStatusList}, nil -} - -// SetUserStatus Synchronize user's online status. -func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { - err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) - if err != nil { - return nil, err - } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - - return &pbuser.SetUserStatusResp{}, nil -} - -// GetSubscribeUsersStatus Get the online status of subscribers. -func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.db.GetUserStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil -} - // ProcessUserCommandAdd user general function add. func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.ProcessUserCommandAddReq) (*pbuser.ProcessUserCommandAddResp, error) { err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) @@ -736,8 +669,3 @@ func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (* } return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } - -func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { - //TODO implement me - panic("implement me") -} diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694b..1ef4943cd6 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -20,6 +20,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" @@ -39,7 +40,7 @@ type CronTaskConfig struct { } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.RetainChatRecords) + log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } @@ -66,10 +67,31 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearFunc); err != nil { return errs.Wrap(err) } - log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) + + tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return err + } + thirdClient := third.NewThirdClient(tConn) + + deleteFunc := func() { + now := time.Now() + deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + return + } + log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteFunc); err != nil { + return errs.Wrap(err) + } + log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index 6ea6a29ed6..f4deb9fb1b 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -15,7 +15,7 @@ package apistruct import ( - sdkws "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/sdkws" ) // SendMsg defines the structure for sending messages with various metadata. diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 78004094c7..29d3fba330 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -37,6 +37,7 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { ret.configMap = map[string]any{ OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, ShareFileName: &msgGatewayConfig.Share, + RedisConfigFileName: &msgGatewayConfig.RedisConfig, WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index d8916ad04c..6d9c9cc5d8 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -109,8 +109,9 @@ type API struct { } type CronTask struct { - ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` + CronExecuteTime string `mapstructure:"cronExecuteTime"` + RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { diff --git a/pkg/common/ginprometheus/ginprometheus.go b/pkg/common/ginprometheus/ginprometheus.go index c2e6bdcca0..64f8a0d8a4 100644 --- a/pkg/common/ginprometheus/ginprometheus.go +++ b/pkg/common/ginprometheus/ginprometheus.go @@ -14,430 +14,431 @@ package ginprometheus -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var defaultMetricPath = "/metrics" - -// counter, counter_vec, gauge, gauge_vec, -// histogram, histogram_vec, summary, summary_vec. -var ( - reqCounter = &Metric{ - ID: "reqCnt", - Name: "requests_total", - Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", - Type: "counter_vec", - Args: []string{"code", "method", "handler", "host", "url"}} - - reqDuration = &Metric{ - ID: "reqDur", - Name: "request_duration_seconds", - Description: "The HTTP request latencies in seconds.", - Type: "histogram_vec", - Args: []string{"code", "method", "url"}, - } - - resSize = &Metric{ - ID: "resSz", - Name: "response_size_bytes", - Description: "The HTTP response sizes in bytes.", - Type: "summary"} - - reqSize = &Metric{ - ID: "reqSz", - Name: "request_size_bytes", - Description: "The HTTP request sizes in bytes.", - Type: "summary"} - - standardMetrics = []*Metric{ - reqCounter, - reqDuration, - resSize, - reqSize, - } -) - -/* -RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control -the cardinality of the request counter's "url" label, which might be required in some contexts. -For instance, if for a "/customer/:name" route you don't want to generate a time series for every -possible customer name, you could use this function: - - func(c *gin.Context) string { - url := c.Request.URL.Path - for _, p := range c.Params { - if p.Key == "name" { - url = strings.Replace(url, p.Value, ":name", 1) - break - } - } - return url - } - -which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". -*/ -type RequestCounterURLLabelMappingFn func(c *gin.Context) string - -// Metric is a definition for the name, description, type, ID, and -// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. -type Metric struct { - MetricCollector prometheus.Collector - ID string - Name string - Description string - Type string - Args []string -} - -// Prometheus contains the metrics gathered by the instance and its path. -type Prometheus struct { - reqCnt *prometheus.CounterVec - reqDur *prometheus.HistogramVec - reqSz, resSz prometheus.Summary - router *gin.Engine - listenAddress string - Ppg PrometheusPushGateway - - MetricsList []*Metric - MetricsPath string - - ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn - - // gin.Context string to use as a prometheus URL label - URLLabelFromContext string -} - -// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). -type PrometheusPushGateway struct { - - // Push interval in seconds - PushIntervalSeconds time.Duration - - // Push Gateway URL in format http://domain:port - // where JOBNAME can be any string of your choice - PushGatewayURL string - - // Local metrics URL where metrics are fetched from, this could be omitted in the future - // if implemented using prometheus common/expfmt instead - MetricsURL string - - // pushgateway job name, defaults to "gin" - Job string -} - -// NewPrometheus generates a new set of metrics with a certain subsystem name. -func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { - if subsystem == "" { - subsystem = "app" - } - - var metricsList []*Metric - - if len(customMetricsList) > 1 { - panic("Too many args. NewPrometheus( string, ).") - } else if len(customMetricsList) == 1 { - metricsList = customMetricsList[0] - } - metricsList = append(metricsList, standardMetrics...) - - p := &Prometheus{ - MetricsList: metricsList, - MetricsPath: defaultMetricPath, - ReqCntURLLabelMappingFn: func(c *gin.Context) string { - return c.FullPath() // e.g. /user/:id , /user/:id/info - }, - } - - p.registerMetrics(subsystem) - - return p -} - -// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL -// every pushIntervalSeconds. Metrics are fetched from metricsURL. -func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { - p.Ppg.PushGatewayURL = pushGatewayURL - p.Ppg.MetricsURL = metricsURL - p.Ppg.PushIntervalSeconds = pushIntervalSeconds - p.startPushTicker() -} - -// SetPushGatewayJob job name, defaults to "gin". -func (p *Prometheus) SetPushGatewayJob(j string) { - p.Ppg.Job = j -} - -// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the -// same address of the gin engine that is being used. -func (p *Prometheus) SetListenAddress(address string) { - p.listenAddress = address - if p.listenAddress != "" { - p.router = gin.Default() - } -} - -// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of -// your content's access log). -func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { - p.listenAddress = listenAddress - if len(p.listenAddress) > 0 { - p.router = r - } -} - -// SetMetricsPath set metrics paths. -func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, prometheusHandler()) - return nil - } -} - -// SetMetricsPathWithAuth set metrics paths with authentication. -func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return nil - } - -} - -func (p *Prometheus) runServer() error { - return p.router.Run(p.listenAddress) -} - -func (p *Prometheus) getMetrics() []byte { - response, err := http.Get(p.Ppg.MetricsURL) - if err != nil { - return nil - } - - defer response.Body.Close() - - body, _ := io.ReadAll(response.Body) - return body -} - -var hostname, _ = os.Hostname() - -func (p *Prometheus) getPushGatewayURL() string { - if p.Ppg.Job == "" { - p.Ppg.Job = "gin" - } - return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname -} - -func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { - req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) - if err != nil { - return - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - fmt.Println("Error sending to push gateway error:", err.Error()) - } - - resp.Body.Close() -} - -func (p *Prometheus) startPushTicker() { - ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) - go func() { - for range ticker.C { - p.sendMetricsToPushGateway(p.getMetrics()) - } - }() -} - -// NewMetric associates prometheus.Collector based on Metric.Type. -func NewMetric(m *Metric, subsystem string) prometheus.Collector { - var metric prometheus.Collector - switch m.Type { - case "counter_vec": - metric = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "counter": - metric = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "gauge_vec": - metric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "gauge": - metric = prometheus.NewGauge( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "histogram_vec": - metric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "histogram": - metric = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "summary_vec": - metric = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "summary": - metric = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - } - return metric -} - -func (p *Prometheus) registerMetrics(subsystem string) { - for _, metricDef := range p.MetricsList { - metric := NewMetric(metricDef, subsystem) - if err := prometheus.Register(metric); err != nil { - fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) - } - - switch metricDef { - case reqCounter: - p.reqCnt = metric.(*prometheus.CounterVec) - case reqDuration: - p.reqDur = metric.(*prometheus.HistogramVec) - case resSize: - p.resSz = metric.(prometheus.Summary) - case reqSize: - p.reqSz = metric.(prometheus.Summary) - } - metricDef.MetricCollector = metric - } -} - -// Use adds the middleware to a gin engine. -func (p *Prometheus) Use(e *gin.Engine) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPath(e) -} - -// UseWithAuth adds the middleware to a gin engine with BasicAuth. -func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPathWithAuth(e, accounts) -} - -// HandlerFunc defines handler function for middleware. -func (p *Prometheus) HandlerFunc() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.URL.Path == p.MetricsPath { - c.Next() - return - } - - start := time.Now() - reqSz := computeApproximateRequestSize(c.Request) - - c.Next() - - status := strconv.Itoa(c.Writer.Status()) - elapsed := float64(time.Since(start)) / float64(time.Second) - resSz := float64(c.Writer.Size()) - - url := p.ReqCntURLLabelMappingFn(c) - if len(p.URLLabelFromContext) > 0 { - u, found := c.Get(p.URLLabelFromContext) - if !found { - u = "unknown" - } - url = u.(string) - } - p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) - p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() - p.reqSz.Observe(float64(reqSz)) - p.resSz.Observe(resSz) - } -} - -func prometheusHandler() gin.HandlerFunc { - h := promhttp.Handler() - return func(c *gin.Context) { - h.ServeHTTP(c.Writer, c.Request) - } -} - -func computeApproximateRequestSize(r *http.Request) int { - var s int - if r.URL != nil { - s = len(r.URL.Path) - } - - s += len(r.Method) - s += len(r.Proto) - for name, values := range r.Header { - s += len(name) - for _, value := range values { - s += len(value) - } - } - s += len(r.Host) - - // r.FormData and r.MultipartForm are assumed to be included in r.URL. - - if r.ContentLength != -1 { - s += int(r.ContentLength) - } - return s -} +// +//import ( +// "bytes" +// "fmt" +// "io" +// "net/http" +// "os" +// "strconv" +// "time" +// +// "github.com/gin-gonic/gin" +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" +//) +// +//var defaultMetricPath = "/metrics" +// +//// counter, counter_vec, gauge, gauge_vec, +//// histogram, histogram_vec, summary, summary_vec. +//var ( +// reqCounter = &Metric{ +// ID: "reqCnt", +// Name: "requests_total", +// Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", +// Type: "counter_vec", +// Args: []string{"code", "method", "handler", "host", "url"}} +// +// reqDuration = &Metric{ +// ID: "reqDur", +// Name: "request_duration_seconds", +// Description: "The HTTP request latencies in seconds.", +// Type: "histogram_vec", +// Args: []string{"code", "method", "url"}, +// } +// +// resSize = &Metric{ +// ID: "resSz", +// Name: "response_size_bytes", +// Description: "The HTTP response sizes in bytes.", +// Type: "summary"} +// +// reqSize = &Metric{ +// ID: "reqSz", +// Name: "request_size_bytes", +// Description: "The HTTP request sizes in bytes.", +// Type: "summary"} +// +// standardMetrics = []*Metric{ +// reqCounter, +// reqDuration, +// resSize, +// reqSize, +// } +//) +// +///* +//RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control +//the cardinality of the request counter's "url" label, which might be required in some contexts. +//For instance, if for a "/customer/:name" route you don't want to generate a time series for every +//possible customer name, you could use this function: +// +// func(c *gin.Context) string { +// url := c.Request.URL.Path +// for _, p := range c.Params { +// if p.Key == "name" { +// url = strings.Replace(url, p.Value, ":name", 1) +// break +// } +// } +// return url +// } +// +//which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". +//*/ +//type RequestCounterURLLabelMappingFn func(c *gin.Context) string +// +//// Metric is a definition for the name, description, type, ID, and +//// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. +//type Metric struct { +// MetricCollector prometheus.Collector +// ID string +// Name string +// Description string +// Type string +// Args []string +//} +// +//// Prometheus contains the metrics gathered by the instance and its path. +//type Prometheus struct { +// reqCnt *prometheus.CounterVec +// reqDur *prometheus.HistogramVec +// reqSz, resSz prometheus.Summary +// router *gin.Engine +// listenAddress string +// Ppg PrometheusPushGateway +// +// MetricsList []*Metric +// MetricsPath string +// +// ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn +// +// // gin.Context string to use as a prometheus URL label +// URLLabelFromContext string +//} +// +//// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). +//type PrometheusPushGateway struct { +// +// // Push interval in seconds +// PushIntervalSeconds time.Duration +// +// // Push Gateway URL in format http://domain:port +// // where JOBNAME can be any string of your choice +// PushGatewayURL string +// +// // Local metrics URL where metrics are fetched from, this could be omitted in the future +// // if implemented using prometheus common/expfmt instead +// MetricsURL string +// +// // pushgateway job name, defaults to "gin" +// Job string +//} +// +//// NewPrometheus generates a new set of metrics with a certain subsystem name. +//func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { +// if subsystem == "" { +// subsystem = "app" +// } +// +// var metricsList []*Metric +// +// if len(customMetricsList) > 1 { +// panic("Too many args. NewPrometheus( string, ).") +// } else if len(customMetricsList) == 1 { +// metricsList = customMetricsList[0] +// } +// metricsList = append(metricsList, standardMetrics...) +// +// p := &Prometheus{ +// MetricsList: metricsList, +// MetricsPath: defaultMetricPath, +// ReqCntURLLabelMappingFn: func(c *gin.Context) string { +// return c.FullPath() // e.g. /user/:id , /user/:id/info +// }, +// } +// +// p.registerMetrics(subsystem) +// +// return p +//} +// +//// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL +//// every pushIntervalSeconds. Metrics are fetched from metricsURL. +//func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { +// p.Ppg.PushGatewayURL = pushGatewayURL +// p.Ppg.MetricsURL = metricsURL +// p.Ppg.PushIntervalSeconds = pushIntervalSeconds +// p.startPushTicker() +//} +// +//// SetPushGatewayJob job name, defaults to "gin". +//func (p *Prometheus) SetPushGatewayJob(j string) { +// p.Ppg.Job = j +//} +// +//// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the +//// same address of the gin engine that is being used. +//func (p *Prometheus) SetListenAddress(address string) { +// p.listenAddress = address +// if p.listenAddress != "" { +// p.router = gin.Default() +// } +//} +// +//// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of +//// your content's access log). +//func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { +// p.listenAddress = listenAddress +// if len(p.listenAddress) > 0 { +// p.router = r +// } +//} +// +//// SetMetricsPath set metrics paths. +//func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, prometheusHandler()) +// return nil +// } +//} +// +//// SetMetricsPathWithAuth set metrics paths with authentication. +//func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return nil +// } +// +//} +// +//func (p *Prometheus) runServer() error { +// return p.router.Run(p.listenAddress) +//} +// +//func (p *Prometheus) getMetrics() []byte { +// response, err := http.Get(p.Ppg.MetricsURL) +// if err != nil { +// return nil +// } +// +// defer response.Body.Close() +// +// body, _ := io.ReadAll(response.Body) +// return body +//} +// +//var hostname, _ = os.Hostname() +// +//func (p *Prometheus) getPushGatewayURL() string { +// if p.Ppg.Job == "" { +// p.Ppg.Job = "gin" +// } +// return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname +//} +// +//func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { +// req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) +// if err != nil { +// return +// } +// +// client := &http.Client{} +// resp, err := client.Do(req) +// if err != nil { +// fmt.Println("Error sending to push gateway error:", err.Error()) +// } +// +// resp.Body.Close() +//} +// +//func (p *Prometheus) startPushTicker() { +// ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) +// go func() { +// for range ticker.C { +// p.sendMetricsToPushGateway(p.getMetrics()) +// } +// }() +//} +// +//// NewMetric associates prometheus.Collector based on Metric.Type. +//func NewMetric(m *Metric, subsystem string) prometheus.Collector { +// var metric prometheus.Collector +// switch m.Type { +// case "counter_vec": +// metric = prometheus.NewCounterVec( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "counter": +// metric = prometheus.NewCounter( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "gauge_vec": +// metric = prometheus.NewGaugeVec( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "gauge": +// metric = prometheus.NewGauge( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "histogram_vec": +// metric = prometheus.NewHistogramVec( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "histogram": +// metric = prometheus.NewHistogram( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "summary_vec": +// metric = prometheus.NewSummaryVec( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "summary": +// metric = prometheus.NewSummary( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// } +// return metric +//} +// +//func (p *Prometheus) registerMetrics(subsystem string) { +// for _, metricDef := range p.MetricsList { +// metric := NewMetric(metricDef, subsystem) +// if err := prometheus.Register(metric); err != nil { +// fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) +// } +// +// switch metricDef { +// case reqCounter: +// p.reqCnt = metric.(*prometheus.CounterVec) +// case reqDuration: +// p.reqDur = metric.(*prometheus.HistogramVec) +// case resSize: +// p.resSz = metric.(prometheus.Summary) +// case reqSize: +// p.reqSz = metric.(prometheus.Summary) +// } +// metricDef.MetricCollector = metric +// } +//} +// +//// Use adds the middleware to a gin engine. +//func (p *Prometheus) Use(e *gin.Engine) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPath(e) +//} +// +//// UseWithAuth adds the middleware to a gin engine with BasicAuth. +//func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPathWithAuth(e, accounts) +//} +// +//// HandlerFunc defines handler function for middleware. +//func (p *Prometheus) HandlerFunc() gin.HandlerFunc { +// return func(c *gin.Context) { +// if c.Request.URL.Path == p.MetricsPath { +// c.Next() +// return +// } +// +// start := time.Now() +// reqSz := computeApproximateRequestSize(c.Request) +// +// c.Next() +// +// status := strconv.Itoa(c.Writer.Status()) +// elapsed := float64(time.Since(start)) / float64(time.Second) +// resSz := float64(c.Writer.Size()) +// +// url := p.ReqCntURLLabelMappingFn(c) +// if len(p.URLLabelFromContext) > 0 { +// u, found := c.Get(p.URLLabelFromContext) +// if !found { +// u = "unknown" +// } +// url = u.(string) +// } +// p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) +// p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() +// p.reqSz.Observe(float64(reqSz)) +// p.resSz.Observe(resSz) +// } +//} +// +//func prometheusHandler() gin.HandlerFunc { +// h := promhttp.Handler() +// return func(c *gin.Context) { +// h.ServeHTTP(c.Writer, c.Request) +// } +//} +// +//func computeApproximateRequestSize(r *http.Request) int { +// var s int +// if r.URL != nil { +// s = len(r.URL.Path) +// } +// +// s += len(r.Method) +// s += len(r.Proto) +// for name, values := range r.Header { +// s += len(name) +// for _, value := range values { +// s += len(value) +// } +// } +// s += len(r.Host) +// +// // r.FormData and r.MultipartForm are assumed to be included in r.URL. +// +// if r.ContentLength != -1 { +// s += int(r.ContentLength) +// } +// return s +//} diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go new file mode 100644 index 0000000000..95b5c06b68 --- /dev/null +++ b/pkg/common/prommetrics/api.go @@ -0,0 +1,48 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "strconv" +) + +var ( + apiCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_count", + Help: "Total number of API calls", + }, + []string{"path", "method", "code"}, + ) + httpCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_count", + Help: "Total number of HTTP calls", + }, + []string{"path", "method", "status"}, + ) +) + +func ApiInit(prometheusPort int) error { + apiRegistry := prometheus.NewRegistry() + cs := append( + baseCollector, + apiCounter, + httpCounter, + ) + return Init(apiRegistry, prometheusPort, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) +} + +func APICall(path string, method string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "method": method, "code": strconv.Itoa(apiCode)}).Inc() +} + +func HttpCall(path string, method string, status int) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() +} + +//func ApiHandler() http.Handler { +// return promhttp.InstrumentMetricHandler( +// apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), +// ) +//} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go deleted file mode 100644 index 9f2e4c99d7..0000000000 --- a/pkg/common/prommetrics/gin_api.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prommetrics - -import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" - -/* -labels := prometheus.Labels{"label_one": "any", "label_two": "value"} -ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). -*/ -var ( - ApiCustomCnt = &ginprom.Metric{ - Name: "custom_total", - Description: "Custom counter events.", - Type: "counter_vec", - Args: []string{"label_one", "label_two"}, - } -) diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 47e5d02b88..02e408d63b 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,44 +15,24 @@ package prommetrics import ( - gp "github.com/grpc-ecosystem/go-grpc-prometheus" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" + "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "net/http" ) -func NewGrpcPromObj(cusMetrics []prometheus.Collector) (*prometheus.Registry, *gp.ServerMetrics, error) { - reg := prometheus.NewRegistry() - grpcMetrics := gp.NewServerMetrics() - grpcMetrics.EnableHandlingTimeHistogram() - cusMetrics = append(cusMetrics, grpcMetrics, collectors.NewGoCollector()) - reg.MustRegister(cusMetrics...) - return reg, grpcMetrics, nil -} +const commonPath = "/metrics" -func GetGrpcCusMetrics(registerName string, share *config2.Share) []prometheus.Collector { - switch registerName { - case share.RpcRegisterName.MessageGateway: - return []prometheus.Collector{OnlineUserGauge} - case share.RpcRegisterName.Msg: - return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} - case "Transfer": - return []prometheus.Collector{MsgInsertRedisSuccessCounter, MsgInsertRedisFailedCounter, MsgInsertMongoSuccessCounter, MsgInsertMongoFailedCounter, SeqSetFailedCounter} - case share.RpcRegisterName.Push: - return []prometheus.Collector{MsgOfflinePushFailedCounter} - case share.RpcRegisterName.Auth: - return []prometheus.Collector{UserLoginCounter} - default: - return nil +var ( + baseCollector = []prometheus.Collector{ + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), } -} +) -func GetGinCusMetrics(name string) []*ginprometheus.Metric { - switch name { - case "Api": - return []*ginprometheus.Metric{ApiCustomCnt} - default: - return []*ginprometheus.Metric{ApiCustomCnt} - } +func Init(registry *prometheus.Registry, prometheusPort int, path string, handler http.Handler, cs ...prometheus.Collector) error { + registry.MustRegister(cs...) + srv := http.NewServeMux() + srv.Handle(path, handler) + return http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv) } diff --git a/pkg/common/prommetrics/prommetrics_test.go b/pkg/common/prommetrics/prommetrics_test.go index 65b05652f7..14b1aaff32 100644 --- a/pkg/common/prommetrics/prommetrics_test.go +++ b/pkg/common/prommetrics/prommetrics_test.go @@ -14,46 +14,39 @@ package prommetrics -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" -) - -func TestNewGrpcPromObj(t *testing.T) { - // Create a custom metric to pass into the NewGrpcPromObj function. - customMetric := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "test_metric", - Help: "This is a test metric.", - }) - cusMetrics := []prometheus.Collector{customMetric} - - // Call NewGrpcPromObj with the custom metrics. - reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) - - // Assert no error was returned. - assert.NoError(t, err) - - // Assert the registry was correctly initialized. - assert.NotNil(t, reg) - - // Assert the grpcMetrics was correctly initialized. - assert.NotNil(t, grpcMetrics) - - // Assert that the custom metric is registered. - mfs, err := reg.Gather() - assert.NoError(t, err) - assert.NotEmpty(t, mfs) // Ensure some metrics are present. - found := false - for _, mf := range mfs { - if *mf.Name == "test_metric" { - found = true - break - } - } - assert.True(t, found, "Custom metric not found in registry") -} +//func TestNewGrpcPromObj(t *testing.T) { +// // Create a custom metric to pass into the NewGrpcPromObj function. +// customMetric := prometheus.NewCounter(prometheus.CounterOpts{ +// Name: "test_metric", +// Help: "This is a test metric.", +// }) +// cusMetrics := []prometheus.Collector{customMetric} +// +// // Call NewGrpcPromObj with the custom metrics. +// reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) +// +// // Assert no error was returned. +// assert.NoError(t, err) +// +// // Assert the registry was correctly initialized. +// assert.NotNil(t, reg) +// +// // Assert the grpcMetrics was correctly initialized. +// assert.NotNil(t, grpcMetrics) +// +// // Assert that the custom metric is registered. +// mfs, err := reg.Gather() +// assert.NoError(t, err) +// assert.NotEmpty(t, mfs) // Ensure some metrics are present. +// found := false +// for _, mf := range mfs { +// if *mf.Name == "test_metric" { +// found = true +// break +// } +// } +// assert.True(t, found, "Custom metric not found in registry") +//} //func TestGetGrpcCusMetrics(t *testing.T) { // conf := config2.NewGlobalConfig() diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go new file mode 100644 index 0000000000..1da2c1510d --- /dev/null +++ b/pkg/common/prommetrics/rpc.go @@ -0,0 +1,58 @@ +package prommetrics + +import ( + gp "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "strconv" +) + +const rpcPath = commonPath + +var ( + grpcMetrics *gp.ServerMetrics + rpcCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_count", + Help: "Total number of RPC calls", + }, + []string{"name", "path", "code"}, + ) +) + +func RpcInit(cs []prometheus.Collector, prometheusPort int) error { + reg := prometheus.NewRegistry() + cs = append(append( + baseCollector, + rpcCounter, + ), cs...) + return Init(reg, prometheusPort, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) +} + +func RPCCall(name string, path string, code int) { + rpcCounter.With(prometheus.Labels{"name": name, "path": path, "code": strconv.Itoa(code)}).Inc() +} + +func GetGrpcServerMetrics() *gp.ServerMetrics { + if grpcMetrics == nil { + grpcMetrics = gp.NewServerMetrics() + grpcMetrics.EnableHandlingTimeHistogram() + } + return grpcMetrics +} + +func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Collector { + switch registerName { + case share.RpcRegisterName.MessageGateway: + return []prometheus.Collector{OnlineUserGauge} + case share.RpcRegisterName.Msg: + return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} + case share.RpcRegisterName.Push: + return []prometheus.Collector{MsgOfflinePushFailedCounter} + case share.RpcRegisterName.Auth: + return []prometheus.Collector{UserLoginCounter} + default: + return nil + } +} diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index 197b6f7fc8..f0abb82856 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -16,6 +16,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -40,3 +41,16 @@ var ( Help: "The number of failed set seq", }) ) + +func TransferInit(prometheusPort int) error { + reg := prometheus.NewRegistry() + cs := append( + baseCollector, + MsgInsertRedisSuccessCounter, + MsgInsertRedisFailedCounter, + MsgInsertMongoSuccessCounter, + MsgInsertMongoFailedCounter, + SeqSetFailedCounter, + ) + return Init(reg, prometheusPort, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index b531daa47d..4091a5f6e6 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -17,9 +17,9 @@ package startrpc import ( "context" "fmt" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" - "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" "net" "net/http" "os" @@ -29,7 +29,6 @@ import ( "syscall" "time" - grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -38,14 +37,13 @@ import ( "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/network" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, - registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, + registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) @@ -77,13 +75,18 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC return err } - var reg *prometheus.Registry - var metric *grpcprometheus.ServerMetrics + //var reg *prometheus.Registry + //var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append( + options, mw.GrpcServer(), + prommetricsUnaryInterceptor(rpcRegisterName), + prommetricsStreamInterceptor(rpcRegisterName), + ) } else { options = append(options, mw.GrpcServer()) } @@ -122,13 +125,18 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC netDone <- struct{}{} return } - metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } + //metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + //} }() } @@ -175,3 +183,25 @@ func gracefulStopWithCtx(ctx context.Context, f func()) error { return nil } } + +func prommetricsUnaryInterceptor(rpcRegisterName string) grpc.ServerOption { + getCode := func(err error) int { + if err == nil { + return 0 + } + rpcErr, ok := err.(interface{ GRPCStatus() *status.Status }) + if !ok { + return -1 + } + return int(rpcErr.GRPCStatus().Code()) + } + return grpc.ChainUnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + resp, err := handler(ctx, req) + prommetrics.RPCCall(rpcRegisterName, info.FullMethod, getCode(err)) + return resp, err + }) +} + +func prommetricsStreamInterceptor(rpcRegisterName string) grpc.ServerOption { + return grpc.ChainStreamInterceptor() +} diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go new file mode 100644 index 0000000000..164e5f2f46 --- /dev/null +++ b/pkg/common/storage/cache/cachekey/online.go @@ -0,0 +1,13 @@ +package cachekey + +import "time" + +const ( + OnlineKey = "ONLINE:" + OnlineChannel = "online_change" + OnlineExpire = time.Hour / 2 +) + +func GetOnlineKey(userID string) string { + return OnlineKey + userID +} diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index 3f0ce98a4d..b32e78300c 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -1,38 +1,30 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cachekey const ( - maxSeq = "MAX_SEQ:" - minSeq = "MIN_SEQ:" - conversationUserMinSeq = "CON_USER_MIN_SEQ:" - hasReadSeq = "HAS_READ_SEQ:" + MallocSeq = "MALLOC_SEQ:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" + + SeqUserMaxSeq = "SEQ_USER_MAX:" + SeqUserMinSeq = "SEQ_USER_MIN:" + SeqUserReadSeq = "SEQ_USER_READ:" ) -func GetMaxSeqKey(conversationID string) string { - return maxSeq + conversationID +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID } -func GetMinSeqKey(conversationID string) string { - return minSeq + conversationID +func GetSeqUserMaxSeqKey(conversationID string, userID string) string { + return SeqUserMaxSeq + conversationID + ":" + userID } -func GetHasReadSeqKey(conversationID string, userID string) string { - return hasReadSeq + userID + ":" + conversationID +func GetSeqUserMinSeqKey(conversationID string, userID string) string { + return SeqUserMinSeq + conversationID + ":" + userID } -func GetConversationUserMinSeqKey(conversationID, userID string) string { - return conversationUserMinSeq + conversationID + "u:" + userID +func GetSeqUserReadSeqKey(conversationID string, userID string) string { + return SeqUserReadSeq + conversationID + ":" + userID } diff --git a/pkg/common/storage/cache/cachekey/user.go b/pkg/common/storage/cache/cachekey/user.go index 7d06d4f751..473ca1b12f 100644 --- a/pkg/common/storage/cache/cachekey/user.go +++ b/pkg/common/storage/cache/cachekey/user.go @@ -17,7 +17,6 @@ package cachekey const ( UserInfoKey = "USER_INFO:" UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:" - olineStatusKey = "ONLINE_STATUS:" ) func GetUserInfoKey(userID string) string { @@ -27,7 +26,3 @@ func GetUserInfoKey(userID string) string { func GetUserGlobalRecvMsgOptKey(userID string) string { return UserGlobalRecvMsgOptKey + userID } - -func GetOnlineStatusKey(modKey string) string { - return olineStatusKey + modKey -} diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 73479bb1bb..91953d9f9a 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -36,7 +36,6 @@ type GroupCache interface { DelGroupMembersHash(groupID string) GroupCache GetGroupMemberIDs(ctx context.Context, groupID string) (groupMemberIDs []string, err error) - GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (groupMemberIDs map[string][]string, err error) DelGroupMemberIDs(groupID string) GroupCache diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go new file mode 100644 index 0000000000..7669c8a118 --- /dev/null +++ b/pkg/common/storage/cache/online.go @@ -0,0 +1,8 @@ +package cache + +import "context" + +type OnlineCache interface { + GetOnline(ctx context.Context, userID string) ([]int32, error) + SetUserOnline(ctx context.Context, userID string, online, offline []int32) error +} diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go new file mode 100644 index 0000000000..5f9a8c82db --- /dev/null +++ b/pkg/common/storage/cache/redis/batch.go @@ -0,0 +1,94 @@ +package redis + +import ( + "context" + "encoding/json" + "github.com/dtm-labs/rockscache" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + "time" + "unsafe" +) + +func getRocksCacheRedisClient(cli *rockscache.Client) redis.UniversalClient { + type Client struct { + rdb redis.UniversalClient + _ rockscache.Options + _ singleflight.Group + } + return (*Client)(unsafe.Pointer(cli)).rdb +} + +func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v *V) K, fn func(ctx context.Context, ids []K) ([]*V, error)) ([]*V, error) { + if len(ids) == 0 { + return nil, nil + } + findKeys := make([]string, 0, len(ids)) + keyId := make(map[string]K) + for _, id := range ids { + key := idKey(id) + if _, ok := keyId[key]; ok { + continue + } + keyId[key] = id + findKeys = append(findKeys, key) + } + slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), findKeys) + if err != nil { + return nil, err + } + result := make([]*V, 0, len(findKeys)) + for _, keys := range slotKeys { + indexCache, err := rcClient.FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { + queryIds := make([]K, 0, len(idx)) + idIndex := make(map[K]int) + for _, index := range idx { + id := keyId[keys[index]] + idIndex[id] = index + queryIds = append(queryIds, id) + } + values, err := fn(ctx, queryIds) + if err != nil { + return nil, err + } + if len(values) == 0 { + return map[int]string{}, nil + } + cacheIndex := make(map[int]string) + for _, value := range values { + id := vId(value) + index, ok := idIndex[id] + if !ok { + continue + } + bs, err := json.Marshal(value) + if err != nil { + return nil, err + } + cacheIndex[index] = string(bs) + } + return cacheIndex, nil + }) + if err != nil { + return nil, err + } + for index, data := range indexCache { + if data == "" { + continue + } + var value V + if err := json.Unmarshal([]byte(data), &value); err != nil { + return nil, err + } + if cb, ok := any(&value).(BatchCacheCallback[K]); ok { + cb.BatchCache(keyId[keys[index]]) + } + result = append(result, &value) + } + } + return result, nil +} + +type BatchCacheCallback[K comparable] interface { + BatchCache(id K) +} diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 95f6699047..52e046a408 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -23,7 +23,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw/specialerror" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" @@ -147,30 +146,30 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin return t, nil } -func batchGetCache[T any, K comparable]( - ctx context.Context, - rcClient *rockscache.Client, - expire time.Duration, - keys []K, - keyFn func(key K) string, - fns func(ctx context.Context, key K) (T, error), -) ([]T, error) { - if len(keys) == 0 { - return nil, nil - } - res := make([]T, 0, len(keys)) - for _, key := range keys { - val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { - return fns(ctx, key) - }) - if err != nil { - if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { - continue - } - return nil, errs.Wrap(err) - } - res = append(res, val) - } - - return res, nil -} +//func batchGetCache[T any, K comparable]( +// ctx context.Context, +// rcClient *rockscache.Client, +// expire time.Duration, +// keys []K, +// keyFn func(key K) string, +// fns func(ctx context.Context, key K) (T, error), +//) ([]T, error) { +// if len(keys) == 0 { +// return nil, nil +// } +// res := make([]T, 0, len(keys)) +// for _, key := range keys { +// val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { +// return fns(ctx, key) +// }) +// if err != nil { +// if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { +// continue +// } +// return nil, errs.Wrap(err) +// } +// res = append(res, val) +// } +// +// return res, nil +//} diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go new file mode 100644 index 0000000000..e4caa2a21f --- /dev/null +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -0,0 +1,55 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "testing" +) + +func TestName(t *testing.T) { + //var rocks rockscache.Client + //rdb := getRocksCacheRedisClient(&rocks) + //t.Log(rdb == nil) + + ctx := context.Background() + rdb, err := redisutil.NewRedisClient(ctx, (&config.Redis{ + Address: []string{"172.16.8.48:16379"}, + Password: "openIM123", + DB: 3, + }).Build()) + if err != nil { + panic(err) + } + mgocli, err := mongoutil.NewMongoDB(ctx, (&config.Mongo{ + Address: []string{"172.16.8.48:37017"}, + Database: "openim_v3", + Username: "openIM", + Password: "openIM123", + MaxPoolSize: 100, + MaxRetry: 1, + }).Build()) + if err != nil { + panic(err) + } + //userMgo, err := mgo.NewUserMongo(mgocli.GetDB()) + //if err != nil { + // panic(err) + //} + //rock := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) + mgoSeqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + panic(err) + } + seqUser := NewSeqUserCacheRedis(rdb, mgoSeqUser) + + res, err := seqUser.GetReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) + if err != nil { + panic(err) + } + + t.Log(res) + +} diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index c491d1b947..95e680afb4 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -164,10 +164,12 @@ func (c *ConversationRedisCache) DelConversations(ownerUserID string, conversati } func (c *ConversationRedisCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*model.Conversation, error) { - return batchGetCache(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { + return batchGetCache2(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { return c.getConversationKey(ownerUserID, conversationID) - }, func(ctx context.Context, conversationID string) (*model.Conversation, error) { - return c.conversationDB.Take(ctx, ownerUserID, conversationID) + }, func(conversation *model.Conversation) string { + return conversation.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) { + return c.conversationDB.Find(ctx, ownerUserID, conversationIDs) }) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 01988310c9..be4687794f 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -70,10 +70,6 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } -//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { -// return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) -//} - func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { return cachekey.GetFriendMaxVersionKey(ownerUserID) } @@ -107,16 +103,6 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { -// newGroupCache := f.CloneFriendCache() -// keys := make([]string, 0, len(ownerUserIDs)) -// for _, userID := range ownerUserIDs { -// keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) -// } -// newGroupCache.AddKeys(keys...) -// return newGroupCache -//} - // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { friendIDs, err := f.GetFriendIDs(ctx, ownerUserID) @@ -193,17 +179,6 @@ func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.Fri return newFriendCache } -//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { -// userIDs, err := f.GetFriendIDs(ctx, ownerUserID) -// if err != nil { -// return nil, err -// } -// if len(userIDs) > f.syncCount { -// userIDs = userIDs[:f.syncCount] -// } -// return userIDs, nil -//} - func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 589678c506..d327c218f3 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -118,34 +118,12 @@ func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { return cachekey.GetJoinGroupMaxVersionKey(userID) } -func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { - key := g.getGroupInfoKey(group.GroupID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } - - return 0, errIndex -} - -func (g *GroupCacheRedis) GetGroupMemberIndex(groupMember *model.GroupMember, keys []string) (int, error) { - key := g.getGroupMemberInfoKey(groupMember.GroupID, groupMember.UserID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } - - return 0, errIndex +func (g *GroupCacheRedis) getGroupID(group *model.Group) string { + return group.GroupID } func (g *GroupCacheRedis) GetGroupsInfo(ctx context.Context, groupIDs []string) (groups []*model.Group, err error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { - return g.getGroupInfoKey(groupID) - }, func(ctx context.Context, groupID string) (*model.Group, error) { - return g.groupDB.Take(ctx, groupID) - }) + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, g.getGroupInfoKey, g.getGroupID, g.groupDB.Find) } func (g *GroupCacheRedis) GetGroupInfo(ctx context.Context, groupID string) (group *model.Group, err error) { @@ -233,19 +211,6 @@ func (g *GroupCacheRedis) GetGroupMemberIDs(ctx context.Context, groupID string) }) } -func (g *GroupCacheRedis) GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (map[string][]string, error) { - m := make(map[string][]string) - for _, groupID := range groupIDs { - userIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - m[groupID] = userIDs - } - - return m, nil -} - func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { cache := g.CloneGroupCache() cache.AddKeys(g.getGroupMemberIDsKey(groupID)) @@ -285,10 +250,12 @@ func (g *GroupCacheRedis) GetGroupMemberInfo(ctx context.Context, groupID, userI } func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, userID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.UserID + }, func(ctx context.Context, userIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.Find(ctx, groupID, userIDs) }) } @@ -301,14 +268,6 @@ func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID st return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) } -func (g *GroupCacheRedis) GetAllGroupMemberInfo(ctx context.Context, groupID string) ([]*model.GroupMember, error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) -} - func (g *GroupCacheRedis) DelGroupMembersInfo(groupID string, userIDs ...string) cache.GroupCache { keys := make([]string, 0, len(userIDs)) for _, userID := range userIDs { @@ -388,42 +347,23 @@ func (g *GroupCacheRedis) GetGroupRolesLevelMemberInfo(ctx context.Context, grou return g.GetGroupMembersInfo(ctx, groupID, userIDs) } -func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) (_ []*model.GroupMember, err error) { +func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) { if len(groupIDs) == 0 { + var err error groupIDs, err = g.GetJoinedGroupIDs(ctx, userID) if err != nil { return nil, err } } - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, groupID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.GroupID + }, func(ctx context.Context, groupIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.FindInGroup(ctx, userID, groupIDs) }) } -//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { -// userIDs, err := g.GetGroupMemberIDs(ctx, groupID) -// if err != nil { -// return nil, err -// } -// if len(userIDs) > g.syncCount { -// userIDs = userIDs[:g.syncCount] -// } -// return userIDs, nil -//} -// -//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { -// groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) -// if err != nil { -// return nil, err -// } -// if len(groupIDs) > g.syncCount { -// groupIDs = groupIDs[:g.syncCount] -// } -// return groupIDs, nil -//} - func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { keys := make([]string, 0, len(groupIDs)) for _, groupID := range groupIDs { diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 2d21cfe135..30f367bb75 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -183,5 +183,4 @@ func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, return nil, nil, err } return seqMsgs, failedSeqs, nil - } diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go new file mode 100644 index 0000000000..dc6a5f775b --- /dev/null +++ b/pkg/common/storage/cache/redis/online.go @@ -0,0 +1,89 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +func NewUserOnline(rdb redis.UniversalClient) cache.OnlineCache { + return &userOnline{ + rdb: rdb, + expire: cachekey.OnlineExpire, + channelName: cachekey.OnlineChannel, + } +} + +type userOnline struct { + rdb redis.UniversalClient + expire time.Duration + channelName string +} + +func (s *userOnline) getUserOnlineKey(userID string) string { + return cachekey.GetOnlineKey(userID) +} + +func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, error) { + members, err := s.rdb.ZRangeByScore(ctx, s.getUserOnlineKey(userID), &redis.ZRangeBy{ + Min: strconv.FormatInt(time.Now().Unix(), 10), + Max: "+inf", + }).Result() + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs := make([]int32, 0, len(members)) + for _, member := range members { + val, err := strconv.Atoi(member) + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs = append(platformIDs, int32(val)) + } + return platformIDs, nil +} + +func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { + script := ` + local key = KEYS[1] + local score = ARGV[3] + local num1 = redis.call("ZCARD", key) + redis.call("ZREMRANGEBYSCORE", key, "-inf", ARGV[2]) + for i = 5, tonumber(ARGV[4])+4 do + redis.call("ZREM", key, ARGV[i]) + end + local num2 = redis.call("ZCARD", key) + for i = 5+tonumber(ARGV[4]), #ARGV do + redis.call("ZADD", key, score, ARGV[i]) + end + redis.call("EXPIRE", key, ARGV[1]) + local num3 = redis.call("ZCARD", key) + local change = (num1 ~= num2) or (num2 ~= num3) + if change then + local members = redis.call("ZRANGE", key, 0, -1) + table.insert(members, KEYS[2]) + redis.call("PUBLISH", KEYS[3], table.concat(members, ":")) + return 1 + else + return 0 + end +` + now := time.Now() + argv := make([]any, 0, 2+len(online)+len(offline)) + argv = append(argv, int32(s.expire/time.Second), now.Unix(), now.Add(s.expire).Unix(), int32(len(offline))) + for _, platformID := range offline { + argv = append(argv, platformID) + } + for _, platformID := range online { + argv = append(argv, platformID) + } + keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} + if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + return err + } + return nil +} diff --git a/pkg/common/storage/cache/redis/redis_shard_manager.go b/pkg/common/storage/cache/redis/redis_shard_manager.go index 98d70dabf9..17e5fecf6f 100644 --- a/pkg/common/storage/cache/redis/redis_shard_manager.go +++ b/pkg/common/storage/cache/redis/redis_shard_manager.go @@ -2,6 +2,7 @@ package redis import ( "context" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -109,7 +110,7 @@ func (rsm *RedisShardManager) ProcessKeysBySlot( func groupKeysBySlot(ctx context.Context, redisClient redis.UniversalClient, keys []string) (map[int64][]string, error) { slots := make(map[int64][]string) clusterClient, isCluster := redisClient.(*redis.ClusterClient) - if isCluster { + if isCluster && len(keys) > 1 { pipe := clusterClient.Pipeline() cmds := make([]*redis.IntCmd, len(keys)) for i, key := range keys { @@ -195,3 +196,16 @@ func ProcessKeysBySlot( } return nil } + +func DeleteCacheBySlot(ctx context.Context, rcClient *rockscache.Client, keys []string) error { + switch len(keys) { + case 0: + return nil + case 1: + return rcClient.TagAsDeletedBatch2(ctx, keys) + default: + return ProcessKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), keys, func(ctx context.Context, slot int64, keys []string) error { + return rcClient.TagAsDeletedBatch2(ctx, keys) + }) + } +} diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index 09ad5b6095..0000000000 --- a/pkg/common/storage/cache/redis/seq.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redis - -import ( - "context" - "errors" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" - "github.com/redis/go-redis/v9" - "sync" -) - -func NewSeqCache(rdb redis.UniversalClient) cache.SeqCache { - return &seqCache{rdb: rdb} -} - -type seqCache struct { - rdb redis.UniversalClient -} - -func (c *seqCache) getMaxSeqKey(conversationID string) string { - return cachekey.GetMaxSeqKey(conversationID) -} - -func (c *seqCache) getMinSeqKey(conversationID string) string { - return cachekey.GetMinSeqKey(conversationID) -} - -func (c *seqCache) getHasReadSeqKey(conversationID string, userID string) string { - return cachekey.GetHasReadSeqKey(conversationID, userID) -} - -func (c *seqCache) getConversationUserMinSeqKey(conversationID, userID string) string { - return cachekey.GetConversationUserMinSeqKey(conversationID, userID) -} - -func (c *seqCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error { - return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err()) -} - -func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey func(conversationID string) string) (int64, error) { - val, err := c.rdb.Get(ctx, getkey(conversationID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) { - m = make(map[string]int64, len(items)) - var ( - reverseMap = make(map[string]string, len(items)) - keys = make([]string, len(items)) - lock sync.Mutex - ) - - for i, v := range items { - keys[i] = getkey(v) - reverseMap[getkey(v)] = v - } - - manager := NewRedisShardManager(c.rdb) - if err = manager.ProcessKeysBySlot(ctx, keys, func(ctx context.Context, _ int64, keys []string) error { - res, err := c.rdb.MGet(ctx, keys...).Result() - if err != nil && !errors.Is(err, redis.Nil) { - return errs.Wrap(err) - } - - // len(res) <= len(items) - for i := range res { - strRes, ok := res[i].(string) - if !ok { - continue - } - val := stringutil.StringToInt64(strRes) - if val != 0 { - lock.Lock() - m[reverseMap[keys[i]]] = val - lock.Unlock() - } - } - return nil - }); err != nil { - return nil, err - } - - return m, nil -} - -func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -} - -func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -} - -func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { - for conversationID, seq := range seqs { - if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { - return errs.Wrap(err) - } - } - return nil -} - -func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return c.setSeqs(ctx, seqs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMinSeqKey) -} - -func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, userIDs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getConversationUserMinSeqKey(conversationID, userID), minSeq, 0).Err()) -} - -func (c *seqCache) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(conversationID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getHasReadSeqKey(conversationID, userID), hasReadSeq, 0).Err()) -} - -func (c *seqCache) SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(userID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getHasReadSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, err - } - return val, nil -} diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go new file mode 100644 index 0000000000..76cac2b02f --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -0,0 +1,333 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "time" +) + +func NewSeqConversationCacheRedis(rdb redis.UniversalClient, mgo database.SeqConversation) cache.SeqConversationCache { + return &seqConversationCacheRedis{ + rdb: rdb, + mgo: mgo, + lockTime: time.Second * 3, + dataTime: time.Hour * 24 * 365, + minSeqExpireTime: time.Hour, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqConversationCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqConversation + rocks *rockscache.Client + lockTime time.Duration + dataTime time.Duration + minSeqExpireTime time.Duration +} + +func (s *seqConversationCacheRedis) getMinSeqKey(conversationID string) string { + return cachekey.GetMallocMinSeqKey(conversationID) +} + +func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return s.SetMinSeqs(ctx, map[string]int64{conversationID: seq}) +} + +func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache(ctx, s.rocks, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) +} + +func (s *seqConversationCacheRedis) getSingleMaxSeq(ctx context.Context, conversationID string) (map[string]int64, error) { + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return nil, err + } + return map[string]int64{conversationID: seq}, nil +} + +func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []string, keyConversationID map[string]string, seqs map[string]int64) error { + result := make([]*redis.StringCmd, len(keys)) + pipe := s.rdb.Pipeline() + for i, key := range keys { + result[i] = pipe.HGet(ctx, key, "CURR") + } + if _, err := pipe.Exec(ctx); err != nil { + return errs.Wrap(err) + } + var notFoundKey []string + for i, r := range result { + req, err := r.Int64() + if err == nil { + seqs[keyConversationID[keys[i]]] = req + } else if errors.Is(err, redis.Nil) { + notFoundKey = append(notFoundKey, keys[i]) + } else { + return errs.Wrap(err) + } + } + if len(notFoundKey) > 0 { + conversationID := keyConversationID[notFoundKey[0]] + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return err + } + seqs[conversationID] = seq + } + return nil +} + +func (s *seqConversationCacheRedis) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { + switch len(conversationIDs) { + case 0: + return map[string]int64{}, nil + case 1: + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + keys := make([]string, 0, len(conversationIDs)) + keyConversationID := make(map[string]string, len(conversationIDs)) + for _, conversationID := range conversationIDs { + key := s.getSeqMallocKey(conversationID) + if _, ok := keyConversationID[key]; ok { + continue + } + keys = append(keys, key) + keyConversationID[key] = conversationID + } + if len(keys) == 1 { + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return nil, err + } + seqs := make(map[string]int64, len(conversationIDs)) + for _, keys := range slotKeys { + if err := s.batchGetMaxSeq(ctx, keys, keyConversationID, seqs); err != nil { + return nil, err + } + } + return seqs, nil +} + +func (s *seqConversationCacheRedis) getSeqMallocKey(conversationID string) string { + return cachekey.GetMallocSeqKey(conversationID) +} + +func (s *seqConversationCacheRedis) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { + if lastSeq < currSeq { + return 0, errs.New("lastSeq must be greater than currSeq") + } + // 0: success + // 1: success the lock has expired, but has not been locked by anyone else + // 2: already locked, but not by yourself + script := ` +local key = KEYS[1] +local lockValue = ARGV[1] +local dataSecond = ARGV[2] +local curr_seq = tonumber(ARGV[3]) +local last_seq = tonumber(ARGV[4]) +if redis.call("EXISTS", key) == 0 then + redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) + redis.call("EXPIRE", key, dataSecond) + return 1 +end +if redis.call("HGET", key, "LOCK") ~= lockValue then + return 2 +end +redis.call("HDEL", key, "LOCK") +redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) +redis.call("EXPIRE", key, dataSecond) +return 0 +` + result, err := s.rdb.Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq).Int64() + if err != nil { + return 0, errs.Wrap(err) + } + return result, nil +} + +// malloc size=0 is to get the current seq size>0 is to allocate seq +func (s *seqConversationCacheRedis) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0: success + // 1: need to obtain and lock + // 2: already locked + // 3: exceeded the maximum value and locked + script := ` +local key = KEYS[1] +local size = tonumber(ARGV[1]) +local lockSecond = ARGV[2] +local dataSecond = ARGV[3] +local result = {} +if redis.call("EXISTS", key) == 0 then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 1) + table.insert(result, lockValue) + return result +end +if redis.call("HEXISTS", key, "LOCK") == 1 then + table.insert(result, 2) + return result +end +local curr_seq = tonumber(redis.call("HGET", key, "CURR")) +local last_seq = tonumber(redis.call("HGET", key, "LAST")) +if size == 0 then + redis.call("EXPIRE", key, dataSecond) + table.insert(result, 0) + table.insert(result, curr_seq) + table.insert(result, last_seq) + return result +end +local max_seq = curr_seq + size +if max_seq > last_seq then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("HSET", key, "CURR", last_seq) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 3) + table.insert(result, curr_seq) + table.insert(result, last_seq) + table.insert(result, lockValue) + return result +end +redis.call("HSET", key, "CURR", max_seq) +redis.call("EXPIRE", key, dataSecond) +table.insert(result, 0) +table.insert(result, curr_seq) +table.insert(result, last_seq) +return result +` + result, err := s.rdb.Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second)).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return result, nil +} + +func (s *seqConversationCacheRedis) wait(ctx context.Context) error { + timer := time.NewTimer(time.Second / 4) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *seqConversationCacheRedis) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { + for i := 0; i < 10; i++ { + state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) + if err != nil { + log.ZError(ctx, "set seq cache failed", err, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq, "count", i+1) + if err := s.wait(ctx); err != nil { + return + } + continue + } + switch state { + case 0: // ideal state + case 1: + log.ZWarn(ctx, "set seq cache lock not found", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + case 2: + log.ZWarn(ctx, "set seq cache lock to be held by someone else", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + default: + log.ZError(ctx, "set seq cache lock unknown state", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + } + return + } + log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) +} + +func (s *seqConversationCacheRedis) getMallocSize(conversationID string, size int64) int64 { + if size == 0 { + return 0 + } + var basicSize int64 + if msgprocessor.IsGroupConversationID(conversationID) { + basicSize = 100 + } else { + basicSize = 50 + } + basicSize += size + return basicSize +} + +func (s *seqConversationCacheRedis) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errs.New("size must be greater than 0") + } + key := s.getSeqMallocKey(conversationID) + for i := 0; i < 10; i++ { + states, err := s.malloc(ctx, key, size) + if err != nil { + return 0, err + } + switch states[0] { + case 0: // success + return states[1], nil + case 1: // not found + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) + return seq, nil + case 2: // locked + if err := s.wait(ctx); err != nil { + return 0, err + } + continue + case 3: // exceeded cache max value + currSeq := states[1] + lastSeq := states[2] + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + if lastSeq == seq { + s.setSeqRetry(ctx, key, states[3], currSeq+size, seq+mallocSize) + return currSeq, nil + } else { + log.ZWarn(ctx, "malloc seq not equal cache last seq", nil, "conversationID", conversationID, "currSeq", currSeq, "lastSeq", lastSeq, "mallocSeq", seq) + s.setSeqRetry(ctx, key, states[3], seq+size, seq+mallocSize) + return seq, nil + } + default: + log.ZError(ctx, "malloc seq unknown state", nil, "state", states[0], "conversationID", conversationID, "size", size) + return 0, errs.New(fmt.Sprintf("unknown state: %d", states[0])) + } + } + log.ZError(ctx, "malloc seq retrying still failed", nil, "conversationID", conversationID, "size", size) + return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) +} + +func (s *seqConversationCacheRedis) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + return s.Malloc(ctx, conversationID, 0) +} + +func (s *seqConversationCacheRedis) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + keys = append(keys, s.getMinSeqKey(conversationID)) + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} diff --git a/pkg/common/storage/cache/redis/seq_conversation_test.go b/pkg/common/storage/cache/redis/seq_conversation_test.go new file mode 100644 index 0000000000..1a40624b8c --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation_test.go @@ -0,0 +1,109 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +func newTestSeq() *seqConversationCacheRedis { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqConversationMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return NewSeqConversationCacheRedis(rdb, model).(*seqConversationCacheRedis) +} + +func TestSeq(t *testing.T) { + ts := newTestSeq() + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 128 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 10 + cID := strconv.Itoa(index * 1) + for i := 1; ; i++ { + //first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) + } + } +} + +func TestDel(t *testing.T) { + ts := newTestSeq() + for i := 1; i < 100; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "100", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second) + } +} + +func TestSeqMalloc(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "100")) +} + +func TestMinSeq(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMinSeq(context.Background(), "10000000")) +} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go new file mode 100644 index 0000000000..2ad43eebdb --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -0,0 +1,185 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +func NewSeqUserCacheRedis(rdb redis.UniversalClient, mgo database.SeqUser) cache.SeqUser { + return &seqUserCacheRedis{ + rdb: rdb, + mgo: mgo, + readSeqWriteRatio: 100, + expireTime: time.Hour * 24 * 7, + readExpireTime: time.Hour * 24 * 30, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqUserCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqUser + rocks *rockscache.Client + expireTime time.Duration + readExpireTime time.Duration + readSeqWriteRatio int64 +} + +func (s *seqUserCacheRedis) getSeqUserMaxSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMaxSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserMinSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMinSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserReadSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetMaxSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.SetMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) +} + +func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if seq%s.readSeqWriteRatio == 0 { + if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + return errs.Wrap(err) + } + return nil +} + +func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + keys = append(keys, s.getSeqUserMinSeqKey(conversationID, userID)) + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} + +func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + keySeq := make(map[string]int64) + for conversationID, seq := range seqs { + key := s.getSeqUserReadSeqKey(conversationID, userID) + keys = append(keys, key) + keySeq[key] = seq + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return err + } + for _, keys := range slotKeys { + pipe := s.rdb.Pipeline() + for _, key := range keys { + pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10)) + pipe.Expire(ctx, key, s.readExpireTime) + } + if _, err := pipe.Exec(ctx); err != nil { + return err + } + } + return nil +} + +func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + if len(seqs) == 0 { + return nil + } + if err := s.setRedisReadSeqs(ctx, userID, seqs); err != nil { + return err + } + for conversationID, seq := range seqs { + if seq%s.readSeqWriteRatio == 0 { + if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + } + return nil +} + +func (s *seqUserCacheRedis) GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { + res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { + return s.getSeqUserReadSeqKey(conversationID, userID) + }, func(v *readSeqModel) string { + return v.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { + seqs, err := s.mgo.GetReadSeqs(ctx, userID, conversationIDs) + if err != nil { + return nil, err + } + res := make([]*readSeqModel, 0, len(seqs)) + for conversationID, seq := range seqs { + res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq}) + } + return res, nil + }) + if err != nil { + return nil, err + } + data := make(map[string]int64) + for _, v := range res { + data[v.ConversationID] = v.Seq + } + return data, nil +} + +var _ BatchCacheCallback[string] = (*readSeqModel)(nil) + +type readSeqModel struct { + ConversationID string + Seq int64 +} + +func (r *readSeqModel) BatchCache(conversationID string) { + r.ConversationID = conversationID +} + +func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) { + r.Seq, err = strconv.ParseInt(string(bytes), 10, 64) + return +} + +func (r *readSeqModel) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatInt(r.Seq, 10)), nil +} diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go new file mode 100644 index 0000000000..e4fd95922a --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -0,0 +1,79 @@ +package redis + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/redis/go-redis/v9" + "log" + "strconv" + "sync/atomic" + "testing" + "time" +) + +func newTestOnline() *userOnline { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 0, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &userOnline{rdb: rdb, expire: time.Hour, channelName: "user_online"} +} + +func TestOnline(t *testing.T) { + ts := newTestOnline() + var count atomic.Int64 + for i := 0; i < 64; i++ { + go func(userID string) { + var err error + for i := 0; ; i++ { + if i%2 == 0 { + err = ts.SetUserOnline(context.Background(), userID, []int32{5, 6}, []int32{7, 8, 9}) + } else { + err = ts.SetUserOnline(context.Background(), userID, []int32{1, 2, 3}, []int32{4, 5, 6}) + } + if err != nil { + panic(err) + } + count.Add(1) + } + }(strconv.Itoa(10000 + i)) + } + + ticker := time.NewTicker(time.Second) + for range ticker.C { + t.Log(count.Swap(0)) + } +} + +func TestGetOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pIDs, err := ts.GetOnline(ctx, "10000") + if err != nil { + panic(err) + } + t.Log(pIDs) +} + +func TestRecvOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) + + _, err := pubsub.Receive(ctx) + if err != nil { + log.Fatalf("Could not subscribe: %v", err) + } + + ch := pubsub.Channel() + + for msg := range ch { + fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) + } +} diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index 3de01563bf..f6b4907302 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -16,21 +16,14 @@ package redis import ( "context" - "encoding/json" - "errors" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "hash/crc32" - "strconv" "time" ) @@ -61,8 +54,8 @@ func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, } } -func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { - return cachekey.GetOnlineStatusKey(modKey) +func (u *UserCacheRedis) getUserID(user *model.User) string { + return user.UserID } func (u *UserCacheRedis) CloneUserCache() cache.UserCache { @@ -90,11 +83,7 @@ func (u *UserCacheRedis) GetUserInfo(ctx context.Context, userID string) (userIn } func (u *UserCacheRedis) GetUsersInfo(ctx context.Context, userIDs []string) ([]*model.User, error) { - return batchGetCache(ctx, u.rcClient, u.expireTime, userIDs, func(userID string) string { - return u.getUserInfoKey(userID) - }, func(ctx context.Context, userID string) (*model.User, error) { - return u.userDB.Take(ctx, userID) - }) + return batchGetCache2(ctx, u.rcClient, u.expireTime, userIDs, u.getUserInfoKey, u.getUserID, u.userDB.Find) } func (u *UserCacheRedis) DelUsersInfo(userIDs ...string) cache.UserCache { @@ -130,174 +119,3 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } - -// GetUserStatus get user status. -func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) - for _, userID := range userIDs { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - var onlineStatus user.OnlineStatus - key := u.getOnlineStatusKey(modKey) - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - // key or field does not exist - userStatus = append(userStatus, &user.OnlineStatus{ - UserID: userID, - Status: constant.Offline, - PlatformIDs: nil, - }) - - continue - } else { - return nil, errs.Wrap(err) - } - } - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return nil, errs.Wrap(err) - } - onlineStatus.UserID = userID - onlineStatus.Status = constant.Online - userStatus = append(userStatus, &onlineStatus) - } - - return userStatus, nil -} - -// SetUserStatus Set the user status and save it in redis. -func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - key := u.getOnlineStatusKey(modKey) - log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) - isNewKey, err := u.rdb.Exists(ctx, key).Result() - if err != nil { - return errs.Wrap(err) - } - if isNewKey == 0 { - if status == constant.Online { - onlineStatus := user.OnlineStatus{ - UserID: userID, - Status: constant.Online, - PlatformIDs: []int32{platformID}, - } - jsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - u.rdb.Expire(ctx, key, userOlineStatusExpireTime) - - return nil - } - } - - isNil := false - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - isNil = true - } else { - return errs.Wrap(err) - } - } - - if status == constant.Offline { - err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) - if err != nil { - return err - } - } else { - err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} - -func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { - if isNil { - log.ZWarn(ctx, "this user not online,maybe trigger order not right", - err, "userStatus", status) - - return nil - } - var onlineStatus user.OnlineStatus - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - var newPlatformIDs []int32 - for _, val := range onlineStatus.PlatformIDs { - if val != platformID { - newPlatformIDs = append(newPlatformIDs, val) - } - } - if newPlatformIDs == nil { - _, err = u.rdb.HDel(ctx, key, userID).Result() - if err != nil { - return errs.Wrap(err) - } - } else { - onlineStatus.PlatformIDs = newPlatformIDs - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} - -func (u *UserCacheRedis) refreshStatusOnline(ctx context.Context, userID string, platformID int32, isNil bool, err error, result, key string) error { - var onlineStatus user.OnlineStatus - if !isNil { - err := json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - onlineStatus.PlatformIDs = RemoveRepeatedElementsInList(append(onlineStatus.PlatformIDs, platformID)) - } else { - onlineStatus.PlatformIDs = append(onlineStatus.PlatformIDs, platformID) - } - onlineStatus.Status = constant.Online - onlineStatus.UserID = userID - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.WrapMsg(err, "json.Marshal failed") - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - - return nil -} - -type Comparable interface { - ~int | ~string | ~float64 | ~int32 -} - -func RemoveRepeatedElementsInList[T Comparable](slc []T) []T { - var result []T - tempMap := map[T]struct{}{} - for _, e := range slc { - if _, found := tempMap[e]; !found { - tempMap[e] = struct{}{} - result = append(result, e) - } - } - - return result -} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go deleted file mode 100644 index 091b318c89..0000000000 --- a/pkg/common/storage/cache/seq.go +++ /dev/null @@ -1,30 +0,0 @@ -package cache - -import ( - "context" -) - -type SeqCache interface { - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - // seqs map: key userID value minSeq - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) - // seqs map: key conversationID value minSeq - SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - // has read seq - SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - // k: user, v: seq - SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error - // k: conversation, v :seq - UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) - GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) -} diff --git a/pkg/common/storage/cache/seq_conversation.go b/pkg/common/storage/cache/seq_conversation.go new file mode 100644 index 0000000000..2c893a5e86 --- /dev/null +++ b/pkg/common/storage/cache/seq_conversation.go @@ -0,0 +1,12 @@ +package cache + +import "context" + +type SeqConversationCache interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + SetMinSeqs(ctx context.Context, seqs map[string]int64) error +} diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go new file mode 100644 index 0000000000..4d0bb4ffab --- /dev/null +++ b/pkg/common/storage/cache/seq_user.go @@ -0,0 +1,15 @@ +package cache + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error + SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error + GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) +} diff --git a/pkg/common/storage/cache/user.go b/pkg/common/storage/cache/user.go index 5101c0b6ce..69a11635ca 100644 --- a/pkg/common/storage/cache/user.go +++ b/pkg/common/storage/cache/user.go @@ -17,7 +17,6 @@ package cache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/user" ) type UserCache interface { @@ -28,6 +27,6 @@ type UserCache interface { DelUsersInfo(userIDs ...string) UserCache GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error) DelUsersGlobalRecvMsgOpt(userIDs ...string) UserCache - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error + //GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) + //SetUserStatus(ctx context.Context, userID string, status, platformID int32) error } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8eb9e8e6fd..32202ac9e9 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -69,26 +69,19 @@ type CommonMsgDatabase interface { DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, seqs []int64) error - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) + //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) + //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) @@ -108,7 +101,7 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -128,29 +121,20 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, - seq: seq, + seqUser: seqUser, + seqConversation: seqConversation, producer: producerToRedis, producerToMongo: producerToMongo, producerToPush: producerToPush, }, nil } -//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) { -// msgDocModel, err := database.NewMsgMongo(database) -// if err != nil { -// return nil, err -// } -// //todo MsgCacheTimeout -// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline) -// seq := cache.NewSeqCache(rdb) -// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig) -//} - type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache - seq cache.SeqCache + seqConversation cache.SeqConversationCache + seqUser cache.SeqUser producer *kafka.Producer producerToMongo *kafka.Producer producerToPush *kafka.Producer @@ -348,12 +332,16 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) } -func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { - log.ZError(ctx, "storage.seq.GetMaxSeq", err) - return 0, false, err +func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } } + return nil +} + +func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() @@ -361,9 +349,12 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa if lenList < 1 { return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() } - if errs.Unwrap(err) == redis.Nil { - isNew = true + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err } + isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) for _, m := range msgs { @@ -379,14 +370,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - - err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq) - if err != nil { - log.ZError(ctx, "storage.seq.SetMaxSeq error", err, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) if err != nil { log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) prommetrics.SeqSetFailedCounter.Inc() @@ -514,12 +498,12 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq > minSeq { @@ -530,8 +514,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end) return 0, 0, nil, nil } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq) @@ -571,11 +555,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin var successMsgs []*sdkws.MsgData log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil { - if err != redis.Nil { - - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } + if err != nil && !errors.Is(err, redis.Nil) { + log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) } successMsgs = append(successMsgs, cachedMsgs...) log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) @@ -595,16 +576,16 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq < minSeq { @@ -648,7 +629,7 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont if minSeq == 0 { return nil } - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) { @@ -693,12 +674,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + currentUserMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -796,89 +777,40 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st return nil } -func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationID string, seqs []int64) error { - return nil -} - -func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) { - for _, conversationID := range conversationIDs { - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil { - if err == redis.Nil { - log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID) - } else { - log.ZError(ctx, "get max seq failed", err, "conversationID", conversationID) - } - continue - } - if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { - log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1) - } - } -} - -func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) -} - func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMaxSeqs(ctx, conversationIDs) + return db.seqConversation.GetMaxSeqs(ctx, conversationIDs) } func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMaxSeq(ctx, conversationID) + return db.seqConversation.GetMaxSeq(ctx, conversationID) } func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return db.seq.SetMinSeqs(ctx, seqs) -} - -func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMinSeqs(ctx, conversationIDs) -} - -func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMinSeq(ctx, conversationID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) { - return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs) + return db.seqConversation.SetMinSeqs(ctx, seqs) } func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs) + return db.seqUser.SetMinSeqs(ctx, userID, seqs) } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs) + return db.seqUser.SetReadSeqs(ctx, userID, hasReadSeqs) } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq) + return db.seqUser.SetReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs) + return db.seqUser.GetReadSeqs(ctx, userID, conversationIDs) } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seq.GetHasReadSeq(ctx, userID, conversationID) + return db.seqUser.GetReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { @@ -894,11 +826,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context if err != nil { return } - minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID) + minSeqCache, err = db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { return } - maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID) + maxSeqCache, err = db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { return } @@ -1010,33 +942,8 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d } } -//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) { -// var ( -// docNum int -// msgNum int -// start = time.Now() -// ) -// for { -// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return err -// } -// if len(msgs) == 0 { -// return nil -// } -// for _, msg := range msgs { -// num, err := db.deleteOneMsg(ctx, ts, msg) -// if err != nil { -// return err -// } -// docNum++ -// msgNum += num -// } -// } -//} - func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { - dbSeq, err := db.seq.GetMinSeq(ctx, conversationID) + dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { return nil @@ -1046,5 +953,5 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin if dbSeq >= seq { return nil } - return db.seq.SetMinSeq(ctx, conversationID, seq) + return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b0ad612038..9b56661a59 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -16,13 +16,15 @@ package controller import ( "context" - redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path/filepath" "time" + redisCache "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -38,20 +40,27 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + DeleteObject(ctx context.Context, name string) error + DeleteSpecifiedData(ctx context.Context, engine string, name string) error + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + DelS3Key(ctx context.Context, engine string, keys ...string) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { return &s3Database{ - s3: cont.New(redis2.NewS3Cache(rdb, s3), s3), - cache: redis2.NewObjectCacheRedis(rdb, obj), - db: obj, + s3: cont.New(redisCache.NewS3Cache(rdb, s3), s3), + cache: redisCache.NewObjectCacheRedis(rdb, obj), + s3cache: redisCache.NewS3Cache(rdb, s3), + db: obj, } } type s3Database struct { - s3 *cont.Controller - cache cache.ObjectCache - db database.ObjectInfo + s3 *cont.Controller + cache cache.ObjectCache + s3cache cont.S3Cache + db database.ObjectInfo } func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) { @@ -111,3 +120,22 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + + return s.db.FindByExpires(ctx, duration, pagination) +} + +func (s *s3Database) DeleteObject(ctx context.Context, name string) error { + return s.s3.DeleteObject(ctx, name) +} +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { + return s.db.Delete(ctx, engine, name) +} + +func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return s.db.FindNotDelByS3(ctx, key, duration) +} + +func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { + return s.s3cache.DelS3Key(ctx, engine, keys...) +} diff --git a/pkg/common/storage/controller/third.go b/pkg/common/storage/controller/third.go index 344501466b..a9c2ae403c 100644 --- a/pkg/common/storage/controller/third.go +++ b/pkg/common/storage/controller/third.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/db/pagination" diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 9efe535c01..59559537b1 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -70,10 +70,6 @@ type UserDatabase interface { GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) // GetSubscribedList Get all subscribed lists GetSubscribedList(ctx context.Context, userID string) ([]string, error) - // GetUserStatus Get the online status of the user - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - // SetUserStatus Set the user status and store the user status in redis - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -199,7 +195,7 @@ func (u *userDatabase) GetAllUserID(ctx context.Context, pagination pagination.P } func (u *userDatabase) GetUserByID(ctx context.Context, userID string) (user *model.User, err error) { - return u.userDB.Take(ctx, userID) + return u.cache.GetUserInfo(ctx, userID) } // CountTotal Get the total number of users. @@ -246,17 +242,6 @@ func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([] return list, nil } -// GetUserStatus get user status. -func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - onlineStatusList, err := u.cache.GetUserStatus(ctx, userIDs) - return onlineStatusList, err -} - -// SetUserStatus Set the user status and save it in redis. -func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - return u.cache.SetUserStatus(ctx, userID, status, platformID) -} - func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index c272b6ef6f..43a7e6095e 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -28,6 +28,8 @@ type GroupMember interface { UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) + Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) + FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 3eb93a10ec..f89822d3c4 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -153,6 +153,22 @@ func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) ( return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } +func (g *GroupMemberMgo) Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"group_id": groupID} + if len(userIDs) > 0 { + filter["user_id"] = bson.M{"$in": userIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) +} + +func (g *GroupMemberMgo) FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"user_id": userID} + if len(groupIDs) > 0 { + filter["group_id"] = bson.M{"$in": groupIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) +} + func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { return mongoutil.FindOne[*model.GroupMember](ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index df4d10ec4f..4242fbb53a 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -16,10 +16,13 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -68,3 +71,14 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ + "create_time": bson.M{"$lt": duration}, + }, pagination) +} +func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{ + "key": key, + "create_time": bson.M{"$gt": duration}, + }) +} diff --git a/pkg/common/storage/database/mgo/seq_conversation.go b/pkg/common/storage/database/mgo/seq_conversation.go new file mode 100644 index 0000000000..7971b7e1a7 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -0,0 +1,103 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqConversationMongo{coll: coll}, nil +} + +type seqConversationMongo struct { + coll *mongo.Collection +} + +func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "conversation_id": conversationID, + } + insert := bson.M{ + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errors.New("size must be greater than 0") + } + if size == 0 { + return s.GetMaxSeq(ctx, conversationID) + } + filter := map[string]any{"conversation_id": conversationID} + update := map[string]any{ + "$inc": map[string]any{"max_seq": size}, + "$set": map[string]any{"min_seq": int64(0)}, + } + opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) + lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) + if err != nil { + return 0, err + } + return lastSeq - size, nil +} + +func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "max_seq") +} + +func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "min_seq") +} + +func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { + return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID}) +} diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go new file mode 100644 index 0000000000..5167314da9 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -0,0 +1,37 @@ +package mgo + +import ( + "context" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Mongodb() *mongo.Database { + return Result( + mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)), + ).Database("openim_v3") +} + +func TestUserSeq(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4)) +} + +func TestConversationSeq(t *testing.T) { + cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo) + t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10)) + t.Log(cSeq.Malloc(context.Background(), "2000", 10)) + t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) +} diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go new file mode 100644 index 0000000000..e0cbb08d9c --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -0,0 +1,110 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { + coll := db.Collection(database.SeqUserName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "user_id", Value: 1}, + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqUserMongo{coll: coll}, nil +} + +type seqUserMongo struct { + coll *mongo.Collection +} + +func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + insert := bson.M{ + "user_id": userID, + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + "read_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1}) + seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt) + if err == nil { + return seq, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "max_seq") +} + +func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "max_seq") +} + +func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "min_seq") +} + +func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "min_seq") +} + +func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "read_seq") +} + +func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { + if len(conversationID) == 0 { + return map[string]int64{}, nil + } + filter := bson.M{"user_id": userID, "conversation_id": bson.M{"$in": conversationID}} + opt := options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1, "read_seq": 1}) + seqs, err := mongoutil.Find[*model.SeqUser](ctx, s.coll, filter, opt) + if err != nil { + return nil, err + } + res := make(map[string]int64) + for _, seq := range seqs { + res[seq.ConversationID] = seq.ReadSeq + } + return res, nil +} + +func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "read_seq") +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 3d9eb6a977..748bd844d2 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -15,4 +15,6 @@ const ( LogName = "log" ObjectName = "s3" UserName = "user" + SeqConversationName = "seq" + SeqUserName = "seq_user" ) diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 554f71f35d..8292006a04 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -16,11 +16,16 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go new file mode 100644 index 0000000000..cf93b795f4 --- /dev/null +++ b/pkg/common/storage/database/seq.go @@ -0,0 +1,11 @@ +package database + +import "context" + +type SeqConversation interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go new file mode 100644 index 0000000000..edd3910d02 --- /dev/null +++ b/pkg/common/storage/database/seq_user.go @@ -0,0 +1,13 @@ +package database + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) +} diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go new file mode 100644 index 0000000000..1dc75eff18 --- /dev/null +++ b/pkg/common/storage/model/seq.go @@ -0,0 +1,7 @@ +package model + +type SeqConversation struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} diff --git a/pkg/common/storage/model/seq_user.go b/pkg/common/storage/model/seq_user.go new file mode 100644 index 0000000000..845996bb8b --- /dev/null +++ b/pkg/common/storage/model/seq_user.go @@ -0,0 +1,9 @@ +package model + +type SeqUser struct { + UserID string `bson:"user_id"` + ConversationID string `bson:"conversation_id"` + MinSeq int64 `bson:"min_seq"` + MaxSeq int64 `bson:"max_seq"` + ReadSeq int64 `bson:"read_seq"` +} diff --git a/pkg/localcache/cache.go b/pkg/localcache/cache.go index 0e040ad389..ba849f8926 100644 --- a/pkg/localcache/cache.go +++ b/pkg/localcache/cache.go @@ -31,6 +31,12 @@ type Cache[V any] interface { Stop() } +func LRUStringHash(key string) uint64 { + h := fnv.New64a() + h.Write(*(*[]byte)(unsafe.Pointer(&key))) + return h.Sum64() +} + func New[V any](opts ...Option) Cache[V] { opt := defaultOption() for _, o := range opts { @@ -49,11 +55,7 @@ func New[V any](opts ...Option) Cache[V] { if opt.localSlotNum == 1 { c.local = createSimpleLRU() } else { - c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, func(key string) uint64 { - h := fnv.New64a() - h.Write(*(*[]byte)(unsafe.Pointer(&key))) - return h.Sum64() - }, createSimpleLRU) + c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, LRUStringHash, createSimpleLRU) } if opt.linkSlotNum > 0 { c.link = link.New(opt.linkSlotNum) diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 64280f238f..2fedffc48b 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,6 +20,7 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + SetHas(key K, value V) bool Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index 970ac083e3..d27e670574 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -89,5 +89,15 @@ func (x *ExpirationLRU[K, V]) Del(key K) bool { return ok } +func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &expirationLruItem[V]{value: value}) + return true + } + return false +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index d6e64aae43..e935c687c4 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,6 +88,28 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } +//func (x *LayLRU[K, V]) Set(key K, value V) { +// x.lock.Lock() +// x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +// x.lock.Unlock() +//} +// +//func (x *LayLRU[K, V]) Has(key K) bool { +// x.lock.Lock() +// defer x.lock.Unlock() +// return x.core.Contains(key) +//} + +func (x *LayLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) + return true + } + return false +} + func (x *LayLRU[K, V]) Del(key K) bool { x.lock.Lock() ok := x.core.Remove(key) diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index d034e94d32..4538ca20e4 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -40,6 +40,10 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) SetHas(key K, value V) bool { + return x.slots[x.getIndex(key)].SetHas(key, value) +} + func (x *slotLRU[K, V]) Del(key K) bool { return x.slots[x.getIndex(key)].Del(key) } diff --git a/pkg/localcache/option.go b/pkg/localcache/option.go index 00bb9d0441..7d91aba6c3 100644 --- a/pkg/localcache/option.go +++ b/pkg/localcache/option.go @@ -30,7 +30,7 @@ func defaultOption() *option { localSuccessTTL: time.Minute, localFailedTTL: time.Second * 5, delFn: make([]func(ctx context.Context, key ...string), 0, 2), - target: emptyTarget{}, + target: EmptyTarget{}, } } @@ -123,14 +123,14 @@ func WithDeleteKeyBefore(fn func(ctx context.Context, key ...string)) Option { } } -type emptyTarget struct{} +type EmptyTarget struct{} -func (e emptyTarget) IncrGetHit() {} +func (e EmptyTarget) IncrGetHit() {} -func (e emptyTarget) IncrGetSuccess() {} +func (e EmptyTarget) IncrGetSuccess() {} -func (e emptyTarget) IncrGetFailed() {} +func (e EmptyTarget) IncrGetFailed() {} -func (e emptyTarget) IncrDelHit() {} +func (e EmptyTarget) IncrDelHit() {} -func (e emptyTarget) IncrDelNotFound() {} +func (e EmptyTarget) IncrDelNotFound() {} diff --git a/pkg/msgprocessor/conversation.go b/pkg/msgprocessor/conversation.go index b369269cc8..f8140cc7df 100644 --- a/pkg/msgprocessor/conversation.go +++ b/pkg/msgprocessor/conversation.go @@ -24,6 +24,10 @@ import ( "google.golang.org/protobuf/proto" ) +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "g_") || strings.HasPrefix(conversationID, "sg_") +} + func GetNotificationConversationIDByMsg(msg *sdkws.MsgData) string { switch msg.SessionType { case constant.SingleChatType: diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go new file mode 100644 index 0000000000..5db68d198d --- /dev/null +++ b/pkg/rpccache/online.go @@ -0,0 +1,100 @@ +package rpccache + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" + "math/rand" + "strconv" + "time" +) + +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) *OnlineCache { + x := &OnlineCache{ + user: user, + group: group, + local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + }), + } + go func() { + ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) + for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + continue + } + storageCache := x.setUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + if fn != nil { + fn(ctx, userID, platformIDs) + } + } + }() + return x +} + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + local lru.LRU[string, []int32] +} + +func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + return o.local.Get(userID, func() ([]int32, error) { + return o.user.GetUserOnlinePlatform(ctx, userID) + }) +} + +func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { + platformIDs, err := o.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return false, err + } + return len(platformIDs) > 0, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { + onlineUserIDs := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) + return onlineUserIDs, nil +} + +func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { + userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return nil, err + } + var onlineUserIDs []string + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) + return onlineUserIDs, nil +} + +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { + return o.local.SetHas(userID, platformIDs) +} diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 25a8eb20d4..6126f5891c 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -110,3 +110,18 @@ func (u *UserLocalCache) GetUsersInfoMap(ctx context.Context, userIDs []string) } return users, nil } + +//func (u *UserLocalCache) GetUserOnlinePlatform(ctx context.Context, userID string) (val []int32, err error) { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform req", "userID", userID) +// defer func() { +// if err == nil { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform return", "value", val) +// } else { +// log.ZError(ctx, "UserLocalCache GetUserOnlinePlatform return", err) +// } +// }() +// return localcache.AnyValue[[]int32](u.local.Get(ctx, cachekey.GetOnlineKey(userID), func(ctx context.Context) (any, error) { +// log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) +// return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) +// })) +//} diff --git a/pkg/rpcclient/third.go b/pkg/rpcclient/third.go index 4c71dff6ad..7cdc60d52f 100644 --- a/pkg/rpcclient/third.go +++ b/pkg/rpcclient/third.go @@ -41,3 +41,7 @@ func NewThird(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, grafanaUrl } return &Third{discov: discov, Client: client, conn: conn, GrafanaUrl: grafanaUrl} } +func (t *Third) DeleteOutdatedData(ctx context.Context, expires int64) error { + _, err := t.Client.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expires}) + return err +} diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index aab96603eb..eabe77b942 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -193,3 +193,25 @@ func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) }) return err } + +func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { + if len(userIDs) == 0 { + return nil, nil + } + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs, UserID: u.imAdminUserID[0]}) + if err != nil { + return nil, err + } + return resp.StatusList, nil +} + +func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + resp, err := u.GetUsersOnlinePlatform(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(resp) == 0 { + return nil, nil + } + return resp[0].PlatformIDs, nil +} diff --git a/pkg/util/useronline/split.go b/pkg/util/useronline/split.go new file mode 100644 index 0000000000..c39d31d15b --- /dev/null +++ b/pkg/util/useronline/split.go @@ -0,0 +1,27 @@ +package useronline + +import ( + "errors" + "strconv" + "strings" +) + +func ParseUserOnlineStatus(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil +} diff --git a/start-config.yml b/start-config.yml index a9c412b33f..21436d7a9a 100644 --- a/start-config.yml +++ b/start-config.yml @@ -14,4 +14,5 @@ serviceBinaries: toolBinaries: - check-free-memory - check-component + - seq maxFileDescriptors: 10000 diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go new file mode 100644 index 0000000000..5200ad5fef --- /dev/null +++ b/tools/seq/internal/main.go @@ -0,0 +1,331 @@ +package internal + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "gopkg.in/yaml.v3" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const ( + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" +) + +const ( + batchSize = 100 + dataVersionCollection = "data_version" + seqKey = "seq" + seqVersion = 38 +) + +func readConfig[T any](dir string, name string) (*T, error) { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return nil, err + } + var conf T + if err := yaml.Unmarshal(data, &conf); err != nil { + return nil, err + } + return &conf, nil +} + +func Main(conf string, del time.Duration) error { + redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) + if err != nil { + return err + } + mongodbConfig, err := readConfig[config.Mongo](conf, cmd.MongodbConfigFileName) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, redisConfig.Build()) + if err != nil { + return err + } + mgocli, err := mongoutil.NewMongoDB(ctx, mongodbConfig.Build()) + if err != nil { + return err + } + versionColl := mgocli.GetDB().Collection(dataVersionCollection) + converted, err := CheckVersion(versionColl, seqKey, seqVersion) + if err != nil { + return err + } + if converted { + fmt.Println("[seq] seq data has been converted") + return nil + } + if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { + return err + } + cSeq, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSeq, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSpitHasReadSeq := func(id string) (conversationID string, userID string, err error) { + // HasReadSeq + userID + ":" + conversationID + arr := strings.Split(id, ":") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) + } + userID = arr[0] + conversationID = arr[1] + return + } + uSpitConversationUserMinSeq := func(id string) (conversationID string, userID string, err error) { + // ConversationUserMinSeq + conversationID + "u:" + userID + arr := strings.Split(id, "u:") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) + } + conversationID = arr[0] + userID = arr[1] + return + } + + ts := []*taskSeq{ + { + Prefix: MaxSeq, + GetSeq: cSeq.GetMaxSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: MinSeq, + GetSeq: cSeq.GetMinSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: HasReadSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetReadSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return err + } + return uSeq.SetReadSeq(ctx, conversationID, userID, seq) + }, + }, + { + Prefix: ConversationUserMinSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetMinSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return err + } + return uSeq.SetMinSeq(ctx, conversationID, userID, seq) + }, + }, + } + + cancel() + ctx = context.Background() + + var wg sync.WaitGroup + wg.Add(len(ts)) + + for i := range ts { + go func(task *taskSeq) { + defer wg.Done() + err := seqRedisToMongo(ctx, rdb, task.GetSeq, task.SetSeq, task.Prefix, del, &task.Count) + task.End = time.Now() + task.Error = err + }(ts[i]) + } + start := time.Now() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var buf bytes.Buffer + + printTaskInfo := func(now time.Time) { + buf.Reset() + buf.WriteString(now.Format(time.DateTime)) + buf.WriteString(" \n") + for i := range ts { + task := ts[i] + if task.Error == nil { + if task.End.IsZero() { + buf.WriteString(fmt.Sprintf("[%s] converting %s* count %d", now.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } else { + buf.WriteString(fmt.Sprintf("[%s] success %s* count %d", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } + } else { + buf.WriteString(fmt.Sprintf("[%s] failed %s* count %d error %s", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count), task.Error)) + } + buf.WriteString("\n") + } + fmt.Println(buf.String()) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case s := <-sigs: + return fmt.Errorf("exit by signal %s", s) + case <-done: + errs := make([]error, 0, len(ts)) + for i := range ts { + task := ts[i] + if task.Error != nil { + errs = append(errs, fmt.Errorf("seq %s failed %w", task.Prefix, task.Error)) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + printTaskInfo(time.Now()) + if err := SetVersion(versionColl, seqKey, seqVersion); err != nil { + return fmt.Errorf("set mongodb seq version %w", err) + } + return nil + case now := <-ticker.C: + printTaskInfo(now) + } + } +} + +type taskSeq struct { + Prefix string + Count int64 + Error error + End time.Time + GetSeq func(ctx context.Context, id string) (int64, error) + SetSeq func(ctx context.Context, id string, seq int64) error +} + +func seqRedisToMongo(ctx context.Context, rdb redis.UniversalClient, getSeq func(ctx context.Context, id string) (int64, error), setSeq func(ctx context.Context, id string, seq int64) error, prefix string, delAfter time.Duration, count *int64) error { + var ( + cursor uint64 + keys []string + err error + ) + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { + return err + } + if len(keys) > 0 { + for _, key := range keys { + seqStr, err := rdb.Get(ctx, key).Result() + if err != nil { + return fmt.Errorf("redis get %s failed %w", key, err) + } + seq, err := strconv.Atoi(seqStr) + if err != nil { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + if seq < 0 { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + id := strings.TrimPrefix(key, prefix) + redisSeq := int64(seq) + mongoSeq, err := getSeq(ctx, id) + if err != nil { + return fmt.Errorf("get mongo seq %s failed %w", key, err) + } + if mongoSeq < redisSeq { + if err := setSeq(ctx, id, redisSeq); err != nil { + return fmt.Errorf("set mongo seq %s failed %w", key, err) + } + } + if delAfter > 0 { + if err := rdb.Expire(ctx, key, delAfter).Err(); err != nil { + return fmt.Errorf("redis expire key %s failed %w", key, err) + } + } else { + if err := rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("redis del key %s failed %w", key, err) + } + } + atomic.AddInt64(count, 1) + } + } + if cursor == 0 { + return nil + } + } +} + +func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { + type VersionTable struct { + Key string `bson:"key"` + Value string `bson:"value"` + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + res, err := mongoutil.FindOne[VersionTable](ctx, coll, bson.M{"key": key}) + if err == nil { + ver, err := strconv.Atoi(res.Value) + if err != nil { + return false, fmt.Errorf("version %s parse error %w", res.Value, err) + } + if ver >= currentVersion { + return true, nil + } + return false, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return false, nil + } else { + return false, err + } +} + +func SetVersion(coll *mongo.Collection, key string, version int) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + option := options.Update().SetUpsert(true) + filter := bson.M{"key": key, "value": strconv.Itoa(version)} + update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}} + return mongoutil.UpdateOne(ctx, coll, filter, update, false, option) +} diff --git a/tools/seq/main.go b/tools/seq/main.go new file mode 100644 index 0000000000..399e6d934e --- /dev/null +++ b/tools/seq/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "time" +) + +func main() { + var ( + config string + second int + ) + flag.StringVar(&config, "c", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") + flag.Parse() + if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { + fmt.Println("seq task", err) + } + fmt.Println("seq task success!") +} From dcc0b5738237b1291ff24444ce393c51b04deecc Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:31:02 +0800 Subject: [PATCH 17/91] Feature domain (#2414) * Revert "fix:log (#2396)" This reverts commit cc2f993e * feat: prometheus config and log --- config/log.yml | 10 ++-------- config/prometheus.yml | 26 +++++++++++++------------- go.mod | 4 ++-- go.sum | 14 ++++++++++---- pkg/common/cmd/root.go | 5 +---- pkg/common/config/config.go | 21 +++++++++------------ 6 files changed, 37 insertions(+), 43 deletions(-) diff --git a/config/log.yml b/config/log.yml index 1f5641622d..2194d89174 100644 --- a/config/log.yml +++ b/config/log.yml @@ -2,14 +2,8 @@ storageLocation: ../../../../logs/ # Log rotation period (in hours), default is acceptable rotationTime: 24 -# Maximum size of each log file (in MB), default is acceptable, it means unlimited -maxSize: 0 -# Number of log files to retain, default is acceptable, it means whenever the log file is rotated, the old log file will be deleted -maxBackups: 10 -# Old log retain time (in days), default is acceptable, it means unlimited -maxAge: 10 -# Whether compress old log files. -compress: false +# Number of log files to retain, default is acceptable +remainRotationCount: 2 # Log level settings: 3 for production environment; 6 for more verbose logging in debugging environments remainLogLevel: 6 # Whether to output to standard output, default is acceptable diff --git a/config/prometheus.yml b/config/prometheus.yml index 99f2e4df34..5db41679f4 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -8,7 +8,7 @@ global: alerting: alertmanagers: - static_configs: - - targets: ['192.168.2.22:19093'] + - targets: ['internal_ip:19093'] # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: @@ -25,59 +25,59 @@ scrape_configs: # prometheus fetches application services - job_name: 'node_exporter' static_configs: - - targets: [ '192.168.2.22:20114' ] + - targets: [ 'internal_ip:20114' ] - job_name: 'openimserver-openim-api' static_configs: - - targets: [ '192.168.2.22:20113' ] + - targets: [ 'internal_ip:20113' ] labels: namespace: 'default' - job_name: 'openimserver-openim-msggateway' static_configs: - - targets: [ '192.168.2.22:20112' ] + - targets: [ 'internal_ip:20112' ] labels: namespace: 'default' - job_name: 'openimserver-openim-msgtransfer' static_configs: - - targets: [ 192.168.2.22:20111, 192.168.2.22:20110, 192.168.2.22:20109, 192.168.2.22:20108 ] + - targets: [ 'internal_ip:20111', 'internal_ip:20110', 'internal_ip:20109', 'internal_ip:20108' ] labels: namespace: 'default' - job_name: 'openimserver-openim-push' static_configs: - - targets: [ '192.168.2.22:20107' ] + - targets: [ 'internal_ip:20107' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-auth' static_configs: - - targets: [ '192.168.2.22:20106' ] + - targets: [ 'internal_ip:20106' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-conversation' static_configs: - - targets: [ '192.168.2.22:20105' ] + - targets: [ 'internal_ip:20105' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-friend' static_configs: - - targets: [ '192.168.2.22:20104' ] + - targets: [ 'internal_ip:20104' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-group' static_configs: - - targets: [ '192.168.2.22:20103' ] + - targets: [ 'internal_ip:20103' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-msg' static_configs: - - targets: [ '192.168.2.22:20102' ] + - targets: [ 'internal_ip:20102' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-third' static_configs: - - targets: [ '192.168.2.22:20101' ] + - targets: [ 'internal_ip:20101' ] labels: namespace: 'default' - job_name: 'openimserver-openim-rpc-user' static_configs: - - targets: [ '192.168.2.22:20100' ] + - targets: [ 'internal_ip:20100' ] labels: namespace: 'default' \ No newline at end of file diff --git a/go.mod b/go.mod index 2e47d1b5e5..a799f2c086 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.30 - github.com/openimsdk/tools v0.0.49-alpha.50 + github.com/openimsdk/tools v0.0.49-alpha.51 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -118,6 +118,7 @@ require ( github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lithammer/shortuuid v3.0.0+incompatible // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -179,7 +180,6 @@ require ( google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gorm.io/gorm v1.25.8 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 649150dec7..d9f9488741 100644 --- a/go.sum +++ b/go.sum @@ -243,6 +243,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -269,6 +271,12 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= @@ -313,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.50 h1:7CaYLVtsBU5kyiTetUOuOkO5FFFmMvSzBEfh2tfCn90= -github.com/openimsdk/tools v0.0.49-alpha.50/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= +github.com/openimsdk/tools v0.0.49-alpha.51 h1:JTPEetVSNOczw1n+XjiPozaH2SBPQAc+9VlPE41wEeY= +github.com/openimsdk/tools v0.0.49-alpha.51/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -587,8 +595,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index f4fcf680bc..08bb6d0642 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -136,11 +136,8 @@ func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { r.log.IsStdout, r.log.IsJson, r.log.StorageLocation, + r.log.RemainRotationCount, r.log.RotationTime, - r.log.MaxBackups, - r.log.MaxSize, - r.log.MaxAge, - r.log.Compress, config.Version, ) if err != nil { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6d9c9cc5d8..f018c7b834 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -19,9 +19,9 @@ import ( "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "github.com/openimsdk/tools/s3/kodo" "strings" "time" ) @@ -42,16 +42,13 @@ type LocalCache struct { } type Log struct { - StorageLocation string `mapstructure:"storageLocation"` - RotationTime uint `mapstructure:"rotationTime"` - RemainLogLevel int `mapstructure:"remainLogLevel"` - MaxSize int `mapstructure:"maxSize"` - MaxBackups int `mapstructure:"maxBackups"` - MaxAge int `mapstructure:"maxAge"` - Compress bool `mapstructure:"compress"` - IsStdout bool `mapstructure:"isStdout"` - IsJson bool `mapstructure:"isJson"` - WithStack bool `mapstructure:"withStack"` + StorageLocation string `mapstructure:"storageLocation"` + RotationTime uint `mapstructure:"rotationTime"` + RemainRotationCount uint `mapstructure:"remainRotationCount"` + RemainLogLevel int `mapstructure:"remainLogLevel"` + IsStdout bool `mapstructure:"isStdout"` + IsJson bool `mapstructure:"isJson"` + WithStack bool `mapstructure:"withStack"` } type Minio struct { @@ -284,7 +281,7 @@ type Third struct { Cos Cos `mapstructure:"cos"` Oss Oss `mapstructure:"oss"` Kodo Kodo `mapstructure:"kodo"` - Aws struct { + Aws struct { Endpoint string `mapstructure:"endpoint"` Region string `mapstructure:"region"` Bucket string `mapstructure:"bucket"` From 42a66cff4a0c1c129ff128d96bca19f4b169be24 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 17 Jul 2024 18:02:14 +0800 Subject: [PATCH 18/91] chore: add ping Handler DEBUG log in msgGateway. (#2415) * chore: add ping Handler debug log in mgsGateway. * update log print content. * update pingHandler method send args. --- internal/msggateway/client.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 446553dc08..2d898ff455 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -96,12 +96,14 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) } -func (c *Client) pingHandler(_ string) error { +func (c *Client) pingHandler(appData string) error { if err := c.conn.SetReadDeadline(pongWait); err != nil { return err } - return c.writePongMsg() + log.ZDebug(c.ctx, "ping Handler Success.", "appData", appData) + + return c.writePongMsg(appData) } func (c *Client) pongHandler(_ string) error { @@ -156,7 +158,7 @@ func (c *Client) readMessage() { return case PingMessage: - err := c.writePongMsg() + err := c.writePongMsg("") log.ZError(c.ctx, "writePongMsg", err) case CloseMessage: @@ -378,7 +380,7 @@ func (c *Client) writePingMsg() error { return c.conn.WriteMessage(PingMessage, nil) } -func (c *Client) writePongMsg() error { +func (c *Client) writePongMsg(appData string) error { if c.closed.Load() { return nil } @@ -391,5 +393,5 @@ func (c *Client) writePongMsg() error { return err } - return c.conn.WriteMessage(PongMessage, nil) + return c.conn.WriteMessage(PongMessage, []byte(appData)) } From 44ecbd776f662c766f5f224845c6845e764bdd82 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 17 Jul 2024 20:33:53 +0800 Subject: [PATCH 19/91] fix: seq conversion bug (#2419) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug --------- Co-authored-by: withchao --- tools/seq/internal/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 5200ad5fef..7f021a90e8 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -116,7 +116,7 @@ func Main(conf string, del time.Duration) error { { Prefix: MaxSeq, GetSeq: cSeq.GetMaxSeq, - SetSeq: cSeq.SetMinSeq, + SetSeq: cSeq.SetMaxSeq, }, { Prefix: MinSeq, From 01f62c8baf77c2e7a4d818c8e29b27fc7f0eded0 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 17 Jul 2024 21:47:02 +0800 Subject: [PATCH 20/91] fix: redis pipe exec (#2421) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec --------- Co-authored-by: withchao --- pkg/common/storage/cache/redis/seq_conversation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index 76cac2b02f..fb8a547dfa 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -63,7 +63,7 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s for i, key := range keys { result[i] = pipe.HGet(ctx, key, "CURR") } - if _, err := pipe.Exec(ctx); err != nil { + if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { return errs.Wrap(err) } var notFoundKey []string From d945a07549b4174114f6b8b607bccf6e2ad7ffad Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:47:55 +0800 Subject: [PATCH 21/91] feat: group members, friends sorting version, client online subscription (#2427) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 23 ++- internal/msggateway/subscription.go | 65 +++--- internal/msggateway/ws_server.go | 4 +- internal/rpc/friend/notification.go | 29 +++ internal/rpc/friend/sync.go | 39 +++- internal/rpc/group/notification.go | 17 +- internal/rpc/group/sync.go | 40 ++-- internal/rpc/user/online.go | 42 +--- internal/rpc/user/user.go | 3 +- pkg/common/storage/controller/user.go | 49 +---- pkg/common/storage/database/mgo/friend.go | 26 ++- .../storage/database/mgo/group_member.go | 15 +- pkg/common/storage/database/mgo/subscribe.go | 189 ------------------ .../storage/database/mgo/version_log.go | 25 ++- .../storage/database/mgo/version_test.go | 14 +- pkg/common/storage/database/subscribe.go | 31 --- pkg/common/storage/model/version_log.go | 5 + tools/seq/main.go | 2 +- 20 files changed, 220 insertions(+), 404 deletions(-) delete mode 100644 pkg/common/storage/database/mgo/subscribe.go delete mode 100644 pkg/common/storage/database/subscribe.go diff --git a/go.mod b/go.mod index a799f2c086..0637492eb5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.30 + github.com/openimsdk/protocol v0.0.69-alpha.38 github.com/openimsdk/tools v0.0.49-alpha.51 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index d9f9488741..92a783c069 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.30 h1:OXzCIpDpIY/GI6h1SDYWN51OS9Xv/BcHaOwq8whPKqI= -github.com/openimsdk/protocol v0.0.69-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.38 h1:kVZCHIXg/el8YJFoIBWhZu1sbbTUqmzgF4l0W3sUH24= +github.com/openimsdk/protocol v0.0.69-alpha.38/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.51 h1:JTPEetVSNOczw1n+XjiPozaH2SBPQAc+9VlPE41wEeY= github.com/openimsdk/tools v0.0.49-alpha.51/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 2d898ff455..889e5c456e 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -75,8 +75,8 @@ type Client struct { token string hbCtx context.Context hbCancel context.CancelFunc - subLock sync.Mutex - subUserIDs map[string]struct{} + subLock *sync.Mutex + subUserIDs map[string]struct{} // client conn subscription list } // ResetClient updates the client's state with new connection and context information. @@ -94,6 +94,11 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closedErr = nil c.token = ctx.GetToken() c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) + c.subLock = new(sync.Mutex) + if c.subUserIDs != nil { + clear(c.subUserIDs) + } + c.subUserIDs = make(map[string]struct{}) } func (c *Client) pingHandler(appData string) error { @@ -246,13 +251,11 @@ func (c *Client) setAppBackgroundStatus(ctx context.Context, req *Req) ([]byte, } func (c *Client) close() { + c.w.Lock() + defer c.w.Unlock() if c.closed.Load() { return } - - c.w.Lock() - defer c.w.Unlock() - c.closed.Store(true) c.conn.Close() c.hbCancel() // Close server-initiated heartbeat. @@ -313,6 +316,14 @@ func (c *Client) KickOnlineMessage() error { return err } +func (c *Client) PushUserOnlineStatus(data []byte) error { + resp := Resp{ + ReqIdentifier: WsSubUserOnlineStatus, + Data: data, + } + return c.writeBinaryMsg(resp) +} + func (c *Client) writeBinaryMsg(resp Resp) error { if c.closed.Load() { return nil diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 9460f5dbfb..9bb41e0dfd 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -2,15 +2,11 @@ package msggateway import ( "context" - "encoding/json" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/idutil" "google.golang.org/protobuf/proto" "sync" - "time" ) func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { @@ -45,33 +41,19 @@ func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, dat return proto.Marshal(&resp) } -type subClient struct { - clients map[string]*Client -} - func newSubscription() *Subscription { return &Subscription{ userIDs: make(map[string]*subClient), } } -type Subscription struct { - lock sync.RWMutex - userIDs map[string]*subClient +type subClient struct { + clients map[string]*Client } -func (s *Subscription) GetClient(userID string) []*Client { - s.lock.RLock() - defer s.lock.RUnlock() - cs, ok := s.userIDs[userID] - if !ok { - return nil - } - clients := make([]*Client, 0, len(cs.clients)) - for _, client := range cs.clients { - clients = append(clients, client) - } - return clients +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient // subscribe to the user's client connection } func (s *Subscription) DelClient(client *Client) { @@ -99,6 +81,20 @@ func (s *Subscription) DelClient(client *Client) { } } +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { if len(addUserIDs)+len(delUserIDs) == 0 { return @@ -121,6 +117,7 @@ func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { continue } client.subUserIDs[userID] = struct{}{} + add[userID] = struct{}{} } client.subLock.Unlock() if len(del)+len(add) == 0 { @@ -154,28 +151,16 @@ func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, p if len(clients) == 0 { return } - msgContent, err := json.Marshal(platformIDs) + onlineStatus, err := proto.Marshal(&sdkws.SubUserOnlineStatusTips{ + Subscribers: []*sdkws.SubUserOnlineStatusElem{{UserID: userID, OnlinePlatformIDs: platformIDs}}, + }) if err != nil { log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) return } - now := time.Now().UnixMilli() - msgID := idutil.GetMsgIDByMD5(userID) - msg := &sdkws.MsgData{ - SendID: userID, - ClientMsgID: msgID, - ServerMsgID: msgID, - SenderPlatformID: constant.AdminPlatformID, - SessionType: constant.NotificationChatType, - ContentType: constant.UserSubscribeOnlineStatusNotification, - Content: msgContent, - SendTime: now, - CreateTime: now, - } for _, client := range clients { - msg.RecvID = client.UserID - if err := client.PushMessage(ctx, msg); err != nil { - log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) + if err := client.PushUserOnlineStatus(onlineStatus); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "changePlatformID", platformIDs) } } } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index e903084a9d..537b8c5f0a 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -358,9 +358,7 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - client.subLock.Lock() - clear(client.subUserIDs) - client.subLock.Unlock() + ws.subscription.DelClient(client) //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index ddee025bb6..5fb34577fc 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -16,6 +16,8 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -191,10 +193,37 @@ func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context f.Notification(ctx, req.OwnerUserID, req.FriendUserID, constant.FriendDeletedNotification, &tips) } +func (f *FriendNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + return + } + } +} + +func (f *FriendNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == relationtb.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (f *FriendNotificationSender) FriendRemarkSetNotification(ctx context.Context, fromUserID, toUserID string) { tips := sdkws.FriendInfoChangedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = fromUserID tips.FromToUserID.ToUserID = toUserID + f.setSortVersion(ctx, &tips.FriendVersion, &tips.FriendVersionID, database.FriendVersionName, toUserID, &tips.FriendSortVersion) f.Notification(ctx, fromUserID, toUserID, constant.FriendRemarkSetNotification, &tips) } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 684894609c..eee9f2afdc 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" + "slices" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -52,12 +53,27 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } + var sortVersion uint64 opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ - Ctx: ctx, - VersionKey: req.UserID, - VersionID: req.VersionID, - VersionNumber: req.Version, - Version: s.db.FindFriendIncrVersion, + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindFriendIncrVersion(ctx, ownerUserID, version, limit) + if err != nil { + return nil, err + } + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == model.VersionSortChangeID { + vl.LogLen-- + sortVersion = uint64(elem.Version) + return true + } + return false + }) + return vl, nil + }, CacheMaxVersion: s.db.FindMaxFriendVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) @@ -65,12 +81,13 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { return &relation.GetIncrementalFriendsResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - Delete: deleteIds, - Insert: insertList, - Update: updateList, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, } }, } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index a8824962dc..a7398795fa 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -306,6 +306,21 @@ func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint6 } } +func (g *GroupNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == model.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -707,7 +722,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.setSortVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID, &tips.GroupSortVersion) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 75d060c0e5..f89a98ee84 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -10,9 +10,13 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" - "slices" ) +func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (*pbgroup.BatchGetIncrementalGroupMemberResp, error) { + //TODO implement me + panic("implement me") +} + func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { @@ -63,7 +67,10 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } - var hasGroupUpdate bool + var ( + hasGroupUpdate bool + sortVersion uint64 + ) opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, VersionKey: req.GroupID, @@ -74,14 +81,20 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { - if elem.EID == "" { + logs := make([]model.VersionLogElem, 0, len(vl.Logs)) + for i, log := range vl.Logs { + switch log.EID { + case model.VersionGroupChangeID: vl.LogLen-- hasGroupUpdate = true - return true + case model.VersionSortChangeID: + vl.LogLen-- + sortVersion = uint64(log.Version) + default: + logs = append(logs, vl.Logs[i]) } - return false - }) + } + vl.Logs = logs if vl.LogLen > 0 { hasGroupUpdate = true } @@ -94,12 +107,13 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - Delete: delIDs, - Insert: insertList, - Update: updateList, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, } }, } diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go index e853ceae23..99b272006f 100644 --- a/internal/rpc/user/online.go +++ b/internal/rpc/user/online.go @@ -3,7 +3,6 @@ package user import ( "context" "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/sdkws" pbuser "github.com/openimsdk/protocol/user" ) @@ -38,23 +37,6 @@ func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) // SubscribeOrCancelUsersStatus Subscribe online or cancel online users. func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { - if req.Genre == constant.SubscriberUser { - err := s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.getUsersOnlineStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err := s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil } @@ -82,34 +64,12 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { return nil, err } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - return &pbuser.SetUserStatusResp{}, nil } // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.getUsersOnlineStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil + return &pbuser.GetSubscribeUsersStatusResp{}, nil } func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 0b96077ec3..779d9b0c4d 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -93,8 +93,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi return err } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) - userMongoDB := mgo.NewUserMongoDriver(mgocli.GetDB()) - database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx(), userMongoDB) + database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 59559537b1..5ce8104e7b 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -62,14 +62,6 @@ type UserDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) - // SubscribeUsersStatus Subscribe a user's presence status - SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // UnsubscribeUsersStatus unsubscribe a user's presence status - UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // GetAllSubscribeList Get a list of all subscriptions - GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) - // GetSubscribedList Get all subscribed lists - GetSubscribedList(ctx context.Context, userID string) ([]string, error) // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -80,14 +72,13 @@ type UserDatabase interface { } type userDatabase struct { - tx tx.Tx - userDB database.User - cache cache.UserCache - mongoDB database.SubscribeUser + tx tx.Tx + userDB database.User + cache cache.UserCache } -func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx, mongoDB database.SubscribeUser) UserDatabase { - return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB} +func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx) UserDatabase { + return &userDatabase{userDB: userDB, cache: cache, tx: tx} } func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error { @@ -212,36 +203,6 @@ func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]stri return u.userDB.SortQuery(ctx, userIDName, asc) } -// SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. -func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) - return err -} - -// UnsubscribeUsersStatus unsubscribe a user's presence status. -func (u *userDatabase) UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.UnsubscriptionList(ctx, userID, userIDs) - return err -} - -// GetAllSubscribeList Get a list of all subscriptions. -func (u *userDatabase) GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetAllSubscribeList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - -// GetSubscribedList Get all subscribed lists. -func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetSubscribedList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 7f456fbdab..76c82bac24 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -109,7 +109,13 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate) + var friendUserIDs []string + if f.IsUpdateIsPinned(args) { + friendUserIDs = []string{model.VersionSortChangeID, friendUserID} + } else { + friendUserIDs = []string{friendUserID} + } + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) }) } @@ -214,7 +220,7 @@ func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ( func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { // Ensure there are IDs to update - if len(friendUserIDs) == 0 { + if len(friendUserIDs) == 0 || len(val) == 0 { return nil // Or return an error if you expect there to always be IDs } @@ -230,7 +236,13 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) + var userIDs []string + if f.IsUpdateIsPinned(val) { + userIDs = append([]string{model.VersionSortChangeID}, friendUserIDs...) + } else { + userIDs = friendUserIDs + } + return f.owner.IncrVersion(ctx, ownerUserID, userIDs, model.VersionStateUpdate) }) } @@ -248,3 +260,11 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) } + +func (f *FriendMgo) IsUpdateIsPinned(data map[string]any) bool { + if data == nil { + return false + } + _, ok := data["is_pinned"] + return ok +} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index f89822d3c4..42b3dd72bd 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -59,7 +59,7 @@ type GroupMemberMgo struct { } func (g *GroupMemberMgo) memberSort() any { - return bson.D{{"role_level", -1}, {"create_time", -1}} + return bson.D{{"role_level", -1}, {"create_time", 1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { @@ -118,7 +118,7 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": bson.M{"role_level": roleLevel}}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, userID}, model.VersionStateUpdate) }) } func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error { @@ -131,10 +131,9 @@ func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID strin bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil { return err } - return nil }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{firstUserID, secondUserID}, model.VersionStateUpdate) + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, firstUserID, secondUserID}, model.VersionStateUpdate) }) } @@ -145,7 +144,13 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + var userIDs []string + if g.IsUpdateRoleLevel(data) { + userIDs = []string{model.VersionSortChangeID, userID} + } else { + userIDs = []string{userID} + } + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/subscribe.go b/pkg/common/storage/database/mgo/subscribe.go deleted file mode 100644 index 5b7d9786b7..0000000000 --- a/pkg/common/storage/database/mgo/subscribe.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - - "github.com/openimsdk/tools/errs" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -// prefixes and suffixes. -const ( - SubscriptionPrefix = "subscription_prefix" - SubscribedPrefix = "subscribed_prefix" -) - -// MaximumSubscription Maximum number of subscriptions. -const ( - MaximumSubscription = 3000 -) - -func NewUserMongoDriver(database *mongo.Database) database.SubscribeUser { - return &UserMongoDriver{ - userCollection: database.Collection(model.SubscribeUserTableName), - } -} - -type UserMongoDriver struct { - userCollection *mongo.Collection -} - -// AddSubscriptionList Subscriber's handling of thresholds. -func (u *UserMongoDriver) AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error { - // Check the number of lists in the key. - pipeline := mongo.Pipeline{ - {{"$match", bson.D{{"user_id", SubscriptionPrefix + userID}}}}, - {{"$project", bson.D{{"count", bson.D{{"$size", "$user_id_list"}}}}}}, - } - // perform aggregate operations - cursor, err := u.userCollection.Aggregate(ctx, pipeline) - if err != nil { - return errs.Wrap(err) - } - defer cursor.Close(ctx) - var cnt struct { - Count int `bson:"count"` - } - // iterate over aggregated results - for cursor.Next(ctx) { - err = cursor.Decode(&cnt) - if err != nil { - return errs.Wrap(err) - } - } - var newUserIDList []string - // If the threshold is exceeded, pop out the previous MaximumSubscription - len(userIDList) and insert it. - if cnt.Count+len(userIDList) > MaximumSubscription { - newUserIDList, err = u.GetAllSubscribeList(ctx, userID) - if err != nil { - return err - } - newUserIDList = newUserIDList[MaximumSubscription-len(userIDList):] - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$set": bson.M{"user_id_list": newUserIDList}}, - ) - if err != nil { - return err - } - // Another way to subscribe to N before pop,Delete after testing - /*for i := 1; i <= MaximumSubscription-len(userIDList); i++ { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{SubscriptionPrefix + userID: bson.M{"$pop": -1}}, - ) - if err != nil { - return err - } - }*/ - } - upsert := true - opts := &options.UpdateOptions{ - Upsert: &upsert, - } - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$addToSet": bson.M{"user_id_list": bson.M{"$each": userIDList}}}, - opts, - ) - if err != nil { - return errs.Wrap(err) - } - for _, user := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + user}, - bson.M{"$addToSet": bson.M{"user_id_list": userID}}, - opts, - ) - if err != nil { - return errs.WrapMsg(err, "transaction failed") - } - } - return nil -} - -// UnsubscriptionList Handling of unsubscribe. -func (u *UserMongoDriver) UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$pull": bson.M{"user_id_list": bson.M{"$in": userIDList}}}, - ) - if err != nil { - return errs.Wrap(err) - } - err = u.RemoveSubscribedListFromUser(ctx, userID, userIDList) - if err != nil { - return errs.Wrap(err) - } - return nil -} - -// RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. -func (u *UserMongoDriver) RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error { - var err error - for _, userIDTemp := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userIDTemp}, - bson.M{"$pull": bson.M{"user_id_list": userID}}, - ) - } - return errs.Wrap(err) -} - -// GetAllSubscribeList Get all users subscribed by this user. -func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} - -// GetSubscribedList Get the user subscribed by those users. -func (u *UserMongoDriver) GetSubscribedList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 8836742f08..3b449007bc 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -18,8 +18,8 @@ import ( func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { lm := &VersionLogMgo{coll: coll} - if lm.initIndex(context.Background()) != nil { - return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) + if err := lm.initIndex(context.Background()); err != nil { + return nil, errs.WrapMsg(err, "init version log index failed", "coll", coll.Name()) } return lm, nil } @@ -33,6 +33,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { Keys: bson.M{ "d_id": 1, }, + Options: options.Index().SetUnique(true), }) return err } @@ -152,8 +153,24 @@ func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []s "$unset": "delete_e_ids", }, } - opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0}) - return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) + projection := bson.M{ + "logs": 0, + } + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(projection) + res, err := mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) + if err != nil { + return nil, err + } + res.Logs = make([]model.VersionLogElem, 0, len(eIds)) + for _, id := range eIds { + res.Logs = append(res.Logs, model.VersionLogElem{ + EID: id, + State: state, + Version: res.Version, + LastUpdate: res.LastUpdate, + }) + } + return res, nil } func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go index 236c61a2ce..4576e45bcd 100644 --- a/pkg/common/storage/database/mgo/version_test.go +++ b/pkg/common/storage/database/mgo/version_test.go @@ -9,12 +9,12 @@ import ( "time" ) -func Result[V any](val V, err error) V { - if err != nil { - panic(err) - } - return val -} +//func Result[V any](val V, err error) V { +// if err != nil { +// panic(err) +// } +// return val +//} func Check(err error) { if err != nil { @@ -30,7 +30,7 @@ func TestName(t *testing.T) { panic(err) } vl := tmp.(*VersionLogMgo) - res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) + res, err := vl.incrVersionResult(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert) if err != nil { t.Log(err) return diff --git a/pkg/common/storage/database/subscribe.go b/pkg/common/storage/database/subscribe.go deleted file mode 100644 index 5905ecd073..0000000000 --- a/pkg/common/storage/database/subscribe.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import "context" - -// SubscribeUser Operation interface of user mongodb. -type SubscribeUser interface { - // AddSubscriptionList Subscriber's handling of thresholds. - AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error - // UnsubscriptionList Handling of unsubscribe. - UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error - // RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. - RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error - // GetAllSubscribeList Get all users subscribed by this user - GetAllSubscribeList(ctx context.Context, id string) (userIDList []string, err error) - // GetSubscribedList Get the user subscribed by those users - GetSubscribedList(ctx context.Context, id string) (userIDList []string, err error) -} diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index 11a40ef24f..6ed8d30f2e 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -14,6 +14,11 @@ const ( VersionStateUpdate ) +const ( + VersionGroupChangeID = "" + VersionSortChangeID = "____S_O_R_T_I_D____" +) + type VersionLogElem struct { EID string `bson:"e_id"` State int32 `bson:"state"` diff --git a/tools/seq/main.go b/tools/seq/main.go index 399e6d934e..16da9f156e 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -12,7 +12,7 @@ func main() { config string second int ) - flag.StringVar(&config, "c", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + flag.StringVar(&config, "c", "", "config directory") flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { From 6c8ac451379d6eb4803076a2b2f02b35aa4455f8 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:08:39 +0800 Subject: [PATCH 22/91] Feat metrics (#2429) * feat: register and alert * feat: grafana --- config/alertmanager.yml | 11 +- config/grafana-template/Demo.json | 5356 +++++++++++++++++++++++++++ config/instance-down-rules.yml | 22 + internal/rpc/user/user.go | 3 + pkg/common/prommetrics/grpc_user.go | 10 + pkg/common/prommetrics/rpc.go | 2 + 6 files changed, 5403 insertions(+), 1 deletion(-) create mode 100644 config/grafana-template/Demo.json create mode 100644 pkg/common/prommetrics/grpc_user.go diff --git a/config/alertmanager.yml b/config/alertmanager.yml index a029448518..6c675ab6f1 100644 --- a/config/alertmanager.yml +++ b/config/alertmanager.yml @@ -11,11 +11,20 @@ templates: - /etc/alertmanager/email.tmpl route: - group_by: ['alertname'] + group_by: [ 'alertname' ] group_wait: 5s group_interval: 5s repeat_interval: 5m receiver: email + routes: + - matchers: + - alertname = "XXX" + group_by: [ 'instance' ] + group_wait: 5s + group_interval: 5s + repeat_interval: 5m + receiver: email + receivers: - name: email email_configs: diff --git a/config/grafana-template/Demo.json b/config/grafana-template/Demo.json new file mode 100644 index 0000000000..dbb11fbf31 --- /dev/null +++ b/config/grafana-template/Demo.json @@ -0,0 +1,5356 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "Is the service up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 9, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "up", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "UP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of online users and login users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "online users" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#37bbff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "online_user_num", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "online users", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "increase(user_login_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "login num", + "range": true, + "refId": "B" + } + ], + "title": "Login Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of register users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "register users" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7437ff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "user_register_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "register users", + "range": true, + "refId": "A" + } + ], + "title": "Register num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of chat msg success.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Success Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of chat msg failed .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "single msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#ff00dc", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "group msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0cffef", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of msg failed offline pushed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 4, + "y": 33 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_offline_push_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "failed msgs", + "range": true, + "refId": "A" + } + ], + "title": "Msg Offline Push Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of failed set seq.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 14, + "y": 33 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(seq_set_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "failed addr: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Seq Set Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of successfully inserted messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Success Insert Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of failed insertion messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Failed Insert Num", + "type": "timeseries" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 22, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of call of all API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (api_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Requests Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of call of all API within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "/friend/get_friend_list" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(api_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Requests Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of err return of API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (api_count{code != \"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Error Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of err return of API with err code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (api_count{code != \"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "API Error Total With Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the qps of API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1ed9d4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(api_count[1m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "qps", + "range": true, + "refId": "A" + } + ], + "title": "API QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of err return of API within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(api_count{code != \"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Error Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of err return of API with err code within the time frame..", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (increase(api_count{code != \"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "API Error Num With Code", + "type": "timeseries" + } + ], + "title": "API", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of call of all RPC.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (rpc_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Total Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the error return of RPC.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (rpc_count{code!=\"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the error return of RPC with code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (rpc_count{code!=\"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Count With Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of call of all RPC within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Total Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of RPC calls within the time frame, aggregated by name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (name) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Num by Name", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of call of RPC within the time frame, aggregated by address.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Num by Address", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the error return of RPC within the time frame within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(rpc_count{code!=\"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the error return of RPC with code within the time frame within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (increase(rpc_count{code!=\"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Num With Code", + "type": "timeseries" + } + ], + "title": "RPC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 25, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of HTTP requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path) (http_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of HTTP requests with status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path, status) (http_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}: {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Count With Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of HTTP requests within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path) (increase(http_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of HTTP requests with status within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path, status) (increase(http_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}: {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Num With Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the qps of HTTP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1ed9d4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(http_count[1m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "qps", + "range": true, + "refId": "A" + } + ], + "title": "HTTP QPS", + "type": "timeseries" + } + ], + "title": "HTTP", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 6, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 9, + "libraryPanel": { + "name": "Virtual Memory bytes", + "uid": "fdriqgnk5lnnke" + }, + "title": "Virtual Memory bytes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 3, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(go_gc_duration_seconds_count{job=~\"$rpcNameFilter\"}[5m]),\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "GC Rate Per Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "expr": "label_replace(\r\n rate(go_gc_duration_seconds_count{job!~\"$rpcNameFilter\"}[5m]),\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "hide": false, + "instant": false, + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "GC Rate Per Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of goroutines.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_goroutines{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of goroutines.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_goroutines{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes allocated and still in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_alloc_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Alloc Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes allocated and still in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_alloc_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Alloc Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes used by the profiling bucket hash table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_buck_hash_sys_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Buck Hash Sys Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes used by the profiling bucket hash table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_buck_hash_sys_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Buck Hash Sys Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes in use by mcache structures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_mcache_inuse_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Mcache Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "description": "This metric represents the number of bytes in use by mcache structures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_mcache_inuse_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Mcache Bytes", + "type": "timeseries" + } + ], + "title": "GO infomation", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "openimserver-openim-rpc.*", + "value": "openimserver-openim-rpc.*" + }, + "hide": 0, + "includeAll": false, + "label": "filter", + "multi": false, + "name": "rpcNameFilter", + "options": [ + { + "selected": true, + "text": "openimserver-openim-rpc.*", + "value": "openimserver-openim-rpc.*" + } + ], + "query": "openimserver-openim-rpc.*", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "{{job}}: {{instance}}", + "value": "{{job}}: {{instance}}" + }, + "description": "common legend name", + "hide": 0, + "includeAll": false, + "label": "legend", + "multi": false, + "name": "legendName", + "options": [ + { + "selected": true, + "text": "{{job}}: {{instance}}", + "value": "{{job}}: {{instance}}" + } + ], + "query": "{{job}}: {{instance}}", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "text": "1h", + "value": "1h" + }, + "description": "Global promQL time range.", + "hide": 0, + "includeAll": false, + "label": "time", + "multi": false, + "name": "time", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "24h", + "value": "24h" + }, + { + "selected": false, + "text": "1w", + "value": "1w" + }, + { + "selected": false, + "text": "4w", + "value": "4w" + }, + { + "selected": false, + "text": "12w", + "value": "12w" + }, + { + "selected": false, + "text": "24w", + "value": "24w" + }, + { + "selected": false, + "text": "1y", + "value": "1y" + }, + { + "selected": false, + "text": "2y", + "value": "2y" + }, + { + "selected": false, + "text": "4y", + "value": "4y" + }, + { + "selected": false, + "text": "10y", + "value": "10y" + } + ], + "query": "1m,5m,30m,1h,3h,6h,12h,24h,1w,4w,12w,24w,1y,2y,4y,10y", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Demo", + "uid": "a506d250-b606-4702-86a7-ac6aa1d069a1", + "version": 22, + "weekStart": "" +} \ No newline at end of file diff --git a/config/instance-down-rules.yml b/config/instance-down-rules.yml index 5541d2c542..bcac7ba60d 100644 --- a/config/instance-down-rules.yml +++ b/config/instance-down-rules.yml @@ -20,3 +20,25 @@ groups: annotations: summary: "Increase in MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter detected" description: "Either MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter has increased in the last 5 minutes, indicating failures in message insert operations to Redis or MongoDB,maybe the redis or mongodb is crash." + + - name: registrations_few + rules: + - alert: RegistrationsFew + expr: increase(user_login_total[1h]) == 0 + for: 1m + labels: + severity: info + annotations: + summary: "Too few registrations within the time frame" + description: "The number of registrations in the last hour is 0. There might be some issues." + + - name: messages_few + rules: + - alert: MessagesFew + expr: (increase(single_chat_msg_process_success_total[1h])+increase(group_chat_msg_process_success_total[1h])) == 0 + for: 1m + labels: + severity: info + annotations: + summary: "Too few messages within the time frame" + description: "The number of messages sent in the last hour is 0. There might be some issues." diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 779d9b0c4d..1e534437df 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -19,6 +19,7 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -310,6 +311,8 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR return nil, err } + prommetrics.UserRegisterCounter.Add(float64(len(users))) + s.webhookAfterUserRegister(ctx, &s.config.WebhooksConfig.AfterUserRegister, req) return resp, nil } diff --git a/pkg/common/prommetrics/grpc_user.go b/pkg/common/prommetrics/grpc_user.go new file mode 100644 index 0000000000..cc2fc42e61 --- /dev/null +++ b/pkg/common/prommetrics/grpc_user.go @@ -0,0 +1,10 @@ +package prommetrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + UserRegisterCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "user_register_total", + Help: "The number of user login", + }) +) diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 1da2c1510d..dc16322dab 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -52,6 +52,8 @@ func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Co return []prometheus.Collector{MsgOfflinePushFailedCounter} case share.RpcRegisterName.Auth: return []prometheus.Collector{UserLoginCounter} + case share.RpcRegisterName.User: + return []prometheus.Collector{UserRegisterCounter} default: return nil } From d0d33b6b78dfe635136091540c84049c10af0f83 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 19 Jul 2024 17:20:11 +0800 Subject: [PATCH 23/91] feat: implement true batchGetIncrGroupMember RPC method and dependency methods. (#2417) * update wip contents. * update protocol pkg. * feat: add BatchOption struct and method. * fix: remove unnecessary field. * feat: implement true BatchGetIncrGroupMember RPC method and corresponding dependency methods. * fix: update mongo version collection have unique index. * optimize method structures. * update resp in add sortVersion field. * fix uncorrect condition. * add errs pkg. --- internal/api/group.go | 42 +--- internal/rpc/conversation/sync.go | 2 +- internal/rpc/friend/sync.go | 1 - internal/rpc/group/sync.go | 154 ++++++++++++- internal/rpc/incrversion/batch_option.go | 207 ++++++++++++++++++ internal/rpc/incrversion/option.go | 11 +- pkg/common/storage/cache/group.go | 1 + pkg/common/storage/cache/redis/group.go | 18 +- pkg/common/storage/controller/group.go | 43 +++- pkg/common/storage/database/group_member.go | 2 + .../storage/database/mgo/group_member.go | 6 + .../storage/database/mgo/version_log.go | 24 +- pkg/common/storage/database/version_log.go | 4 +- 13 files changed, 454 insertions(+), 61 deletions(-) create mode 100644 internal/rpc/incrversion/batch_option.go diff --git a/internal/api/group.go b/internal/api/group.go index e48191ee15..bff0089748 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -19,8 +19,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" - "github.com/openimsdk/tools/apiresp" - "github.com/openimsdk/tools/log" ) type GroupApi rpcclient.Group @@ -148,45 +146,7 @@ func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { } func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { - type BatchIncrementalReq struct { - UserID string `json:"user_id"` - List []*group.GetIncrementalGroupMemberReq `json:"list"` - } - type BatchIncrementalResp struct { - List map[string]*group.GetIncrementalGroupMemberResp `json:"list"` - } - req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq](c) - if err != nil { - apiresp.GinError(c, err) - return - } - resp := &BatchIncrementalResp{ - List: make(map[string]*group.GetIncrementalGroupMemberResp), - } - var ( - changeCount int - ) - for _, req := range req.List { - if _, ok := resp.List[req.GroupID]; ok { - continue - } - res, err := o.Client.GetIncrementalGroupMember(c, req) - if err != nil { - if len(resp.List) == 0 { - apiresp.GinError(c, err) - } else { - log.ZError(c, "group incr sync versopn", err, "groupID", req.GroupID, "success", len(resp.List)) - apiresp.GinSuccess(c, resp) - } - return - } - resp.List[req.GroupID] = res - changeCount += len(res.Insert) + len(res.Delete) + len(res.Update) - if changeCount >= 200 { - break - } - } - apiresp.GinSuccess(c, resp) + a2r.Call(group.GroupClient.BatchGetIncrementalGroupMember, o.Client, c) } func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go index 29c11c4a10..ad88b2bbd5 100644 --- a/internal/rpc/conversation/sync.go +++ b/internal/rpc/conversation/sync.go @@ -2,6 +2,7 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" @@ -40,7 +41,6 @@ func (c *conversationServer) GetIncrementalConversation(ctx context.Context, req Find: func(ctx context.Context, conversationIDs []string) ([]*conversation.Conversation, error) { return c.getConversations(ctx, req.UserID, conversationIDs) }, - ID: func(elem *conversation.Conversation) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*conversation.Conversation, full bool) *conversation.GetIncrementalConversationResp { return &conversation.GetIncrementalConversationResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index eee9f2afdc..145c287da0 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -78,7 +78,6 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) }, - ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index f89a98ee84..0592aa811c 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,6 +2,7 @@ package group import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -10,13 +11,10 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" ) -func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (*pbgroup.BatchGetIncrementalGroupMemberResp, error) { - //TODO implement me - panic("implement me") -} - func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { @@ -104,7 +102,6 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, - ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ VersionID: version.ID.Hex(), @@ -135,6 +132,150 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou return resp, nil } +func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (resp *pbgroup.BatchGetIncrementalGroupMemberResp, err error) { + type VersionInfo struct { + GroupID string + VersionID string + VersionNumber uint64 + } + + var groupIDs []string + + groupsVersionMap := make(map[string]*VersionInfo) + groupsMap := make(map[string]*model.Group) + hasGroupUpdateMap := make(map[string]bool) + sortVersionMap := make(map[string]uint64) + + var targetKeys, versionIDs []string + var versionNumbers []uint64 + + var requestBodyLen int + + for _, group := range req.ReqList { + groupsVersionMap[group.GroupID] = &VersionInfo{ + GroupID: group.GroupID, + VersionID: group.VersionID, + VersionNumber: group.Version, + } + + groupIDs = append(groupIDs, group.GroupID) + } + + groups, err := s.db.FindGroup(ctx, groupIDs) + if err != nil { + return nil, errs.Wrap(err) + } + + for _, group := range groups { + if group.Status == constant.GroupStatusDismissed { + err = servererrs.ErrDismissedAlready.Wrap() + log.ZError(ctx, "This group is Dismissed Already", err, "group is", group.GroupID) + + delete(groupsVersionMap, group.GroupID) + } else { + groupsMap[group.GroupID] = group + } + } + + for groupID, vInfo := range groupsVersionMap { + targetKeys = append(targetKeys, groupID) + versionIDs = append(versionIDs, vInfo.VersionID) + versionNumbers = append(versionNumbers, vInfo.VersionNumber) + } + + opt := incrversion.BatchOption[[]*sdkws.GroupMemberFullInfo, pbgroup.BatchGetIncrementalGroupMemberResp]{ + Ctx: ctx, + TargetKeys: targetKeys, + VersionIDs: versionIDs, + VersionNumbers: versionNumbers, + Versions: func(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { + vLogs, err := s.db.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + for groupID, vlog := range vLogs { + vlogElems := make([]model.VersionLogElem, 0, len(vlog.Logs)) + for i, log := range vlog.Logs { + switch log.EID { + case model.VersionGroupChangeID: + vlog.LogLen-- + hasGroupUpdateMap[groupID] = true + case model.VersionSortChangeID: + vlog.LogLen-- + sortVersionMap[groupID] = uint64(log.Version) + default: + vlogElems = append(vlogElems, vlog.Logs[i]) + } + } + vlog.Logs = vlogElems + if vlog.LogLen > 0 { + hasGroupUpdateMap[groupID] = true + } + } + + return vLogs, nil + }, + CacheMaxVersions: s.db.BatchFindMaxGroupMemberVersionCache, + Find: func(ctx context.Context, groupID string, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + memberInfo, err := s.getGroupMembersInfo(ctx, groupID, ids) + if err != nil { + return nil, err + } + + return memberInfo, err + }, + Resp: func(versions map[string]*model.VersionLog, deleteIdsMap map[string][]string, insertListMap, updateListMap map[string][]*sdkws.GroupMemberFullInfo, fullMap map[string]bool) *pbgroup.BatchGetIncrementalGroupMemberResp { + resList := make(map[string]*pbgroup.GetIncrementalGroupMemberResp) + + for groupID, versionLog := range versions { + resList[groupID] = &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: versionLog.ID.Hex(), + Version: uint64(versionLog.Version), + Full: fullMap[groupID], + Delete: deleteIdsMap[groupID], + Insert: insertListMap[groupID], + Update: updateListMap[groupID], + SortVersion: sortVersionMap[groupID], + } + + requestBodyLen += len(insertListMap[groupID]) + len(updateListMap[groupID]) + len(deleteIdsMap[groupID]) + if requestBodyLen > 200 { + break + } + } + + return &pbgroup.BatchGetIncrementalGroupMemberResp{ + RespList: resList, + } + }, + } + + resp, err = opt.Build() + if err != nil { + return nil, errs.Wrap(err) + } + + for groupID, val := range resp.RespList { + if val.Full || hasGroupUpdateMap[groupID] { + count, err := s.db.FindGroupMemberNum(ctx, groupID) + if err != nil { + return nil, err + } + + owner, err := s.db.TakeGroupOwner(ctx, groupID) + if err != nil { + return nil, err + } + + resp.RespList[groupID].Group = s.groupDB2PB(groupsMap[groupID], owner.UserID, count) + } + } + + return resp, nil + +} + func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err @@ -147,7 +288,6 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. Version: s.db.FindJoinIncrVersion, CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, Find: s.getGroupsInfo, - ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/incrversion/batch_option.go b/internal/rpc/incrversion/batch_option.go new file mode 100644 index 0000000000..34d1b25066 --- /dev/null +++ b/internal/rpc/incrversion/batch_option.go @@ -0,0 +1,207 @@ +package incrversion + +import ( + "context" + "fmt" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type BatchOption[A, B any] struct { + Ctx context.Context + TargetKeys []string + VersionIDs []string + VersionNumbers []uint64 + //SyncLimit int + Versions func(ctx context.Context, dIds []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) + CacheMaxVersions func(ctx context.Context, dIds []string) (map[string]*model.VersionLog, error) + Find func(ctx context.Context, dId string, ids []string) (A, error) + Resp func(versionsMap map[string]*model.VersionLog, deleteIdsMap map[string][]string, insertListMap, updateListMap map[string]A, fullMap map[string]bool) *B +} + +func (o *BatchOption[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *BatchOption[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") + } + if len(o.TargetKeys) == 0 { + return o.newError("targetKeys is empty") + } + if o.Versions == nil { + return o.newError("func versions is nil") + } + if o.Find == nil { + return o.newError("func find is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil +} + +func (o *BatchOption[A, B]) validVersions() []bool { + valids := make([]bool, len(o.VersionIDs)) + for i, versionID := range o.VersionIDs { + objID, err := primitive.ObjectIDFromHex(versionID) + valids[i] = (err == nil && (!objID.IsZero()) && o.VersionNumbers[i] > 0) + } + return valids +} + +func (o *BatchOption[A, B]) equalIDs(objIDs []primitive.ObjectID) []bool { + equals := make([]bool, len(o.VersionIDs)) + for i, versionID := range o.VersionIDs { + equals[i] = versionID == objIDs[i].Hex() + } + return equals +} + +func (o *BatchOption[A, B]) getVersions(tags *[]int) (versions map[string]*model.VersionLog, err error) { + var dIDs []string + var versionNums []uint64 + var limits []int + + valids := o.validVersions() + + if o.CacheMaxVersions == nil { + for i, valid := range valids { + if valid { + (*tags)[i] = tagQuery + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, o.VersionNumbers[i]) + limits = append(limits, syncLimit) + } else { + (*tags)[i] = tagFull + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, 0) + limits = append(limits, 0) + } + } + + versions, err = o.Versions(o.Ctx, dIDs, versionNums, limits) + if err != nil { + return nil, errs.Wrap(err) + } + return versions, nil + + } else { + caches, err := o.CacheMaxVersions(o.Ctx, o.TargetKeys) + if err != nil { + return nil, errs.Wrap(err) + } + + objIDs := make([]primitive.ObjectID, len(o.VersionIDs)) + + for i, versionID := range o.VersionIDs { + objID, _ := primitive.ObjectIDFromHex(versionID) + objIDs[i] = objID + } + + equals := o.equalIDs(objIDs) + for i, valid := range valids { + if !valid { + (*tags)[i] = tagFull + } else if !equals[i] { + (*tags)[i] = tagFull + } else if o.VersionNumbers[i] == uint64(caches[o.TargetKeys[i]].Version) { + (*tags)[i] = tagEqual + } else { + (*tags)[i] = tagQuery + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, o.VersionNumbers[i]) + limits = append(limits, syncLimit) + + delete(caches, o.TargetKeys[i]) + } + } + + if dIDs != nil { + versionMap, err := o.Versions(o.Ctx, dIDs, versionNums, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + for k, v := range versionMap { + caches[k] = v + } + } + + versions = caches + } + return versions, nil +} + +func (o *BatchOption[A, B]) Build() (*B, error) { + if err := o.check(); err != nil { + return nil, errs.Wrap(err) + } + + tags := make([]int, len(o.TargetKeys)) + versions, err := o.getVersions(&tags) + if err != nil { + return nil, errs.Wrap(err) + } + + fullMap := make(map[string]bool) + for i, tag := range tags { + switch tag { + case tagQuery: + vLog := versions[o.TargetKeys[i]] + fullMap[o.TargetKeys[i]] = vLog.ID.Hex() != o.VersionIDs[i] || uint64(vLog.Version) < o.VersionNumbers[i] || len(vLog.Logs) != vLog.LogLen + case tagFull: + fullMap[o.TargetKeys[i]] = true + case tagEqual: + fullMap[o.TargetKeys[i]] = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } + } + + var ( + insertIdsMap = make(map[string][]string) + deleteIdsMap = make(map[string][]string) + updateIdsMap = make(map[string][]string) + ) + + for _, targetKey := range o.TargetKeys { + if !fullMap[targetKey] { + version := versions[targetKey] + insertIds, deleteIds, updateIds := version.DeleteAndChangeIDs() + insertIdsMap[targetKey] = insertIds + deleteIdsMap[targetKey] = deleteIds + updateIdsMap[targetKey] = updateIds + } + } + + var ( + insertListMap = make(map[string]A) + updateListMap = make(map[string]A) + ) + + for targetKey, insertIds := range insertIdsMap { + if len(insertIds) > 0 { + insertList, err := o.Find(o.Ctx, targetKey, insertIds) + if err != nil { + return nil, errs.Wrap(err) + } + insertListMap[targetKey] = insertList + } + } + + for targetKey, updateIds := range updateIdsMap { + if len(updateIds) > 0 { + updateList, err := o.Find(o.Ctx, targetKey, updateIds) + if err != nil { + return nil, errs.Wrap(err) + } + updateListMap[targetKey] = updateList + } + } + + return o.Resp(versions, deleteIdsMap, insertListMap, updateListMap, fullMap), nil +} diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index f7a71244a0..af1200d5c0 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -3,6 +3,7 @@ package incrversion import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson/primitive" @@ -20,7 +21,7 @@ const syncLimit = 200 const ( tagQuery = iota + 1 tagFull - tageEqual + tagEqual ) type Option[A, B any] struct { @@ -33,7 +34,6 @@ type Option[A, B any] struct { Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) //SortID func(ctx context.Context, dId string) ([]string, error) Find func(ctx context.Context, ids []string) ([]A, error) - ID func(elem A) string Resp func(version *model.VersionLog, deleteIds []string, insertList, updateList []A, full bool) *B } @@ -60,9 +60,6 @@ func (o *Option[A, B]) check() error { if o.Find == nil { return o.newError("func find is nil") } - if o.ID == nil { - return o.newError("func id is nil") - } if o.Resp == nil { return o.newError("func resp is nil") } @@ -100,7 +97,7 @@ func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { return cache, nil } if o.VersionNumber == uint64(cache.Version) { - *tag = tageEqual + *tag = tagEqual return cache, nil } *tag = tagQuery @@ -123,7 +120,7 @@ func (o *Option[A, B]) Build() (*B, error) { full = version.ID.Hex() != o.VersionID || uint64(version.Version) < o.VersionNumber || len(version.Logs) != version.LogLen case tagFull: full = true - case tageEqual: + case tagEqual: full = false default: panic(fmt.Errorf("undefined tag %d", tag)) diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 91953d9f9a..1ec0462956 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -64,5 +64,6 @@ type GroupCache interface { DelMaxGroupMemberVersion(groupIDs ...string) GroupCache DelMaxJoinGroupVersion(userIDs ...string) GroupCache FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) + BatchFindMaxGroupMemberVersion(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index d327c218f3..736111df31 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -17,6 +17,8 @@ package redis import ( "context" "fmt" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -28,7 +30,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -390,6 +391,21 @@ func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID }) } +func (g *GroupCacheRedis) BatchFindMaxGroupMemberVersion(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) { + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, + func(groupID string) string { + return g.getGroupMemberMaxVersionKey(groupID) + }, func(versionLog *model.VersionLog) string { + return versionLog.DID + }, func(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) { + // create two slices with len is groupIDs, just need 0 + versions := make([]uint, len(groupIDs)) + limits := make([]int, len(groupIDs)) + + return g.groupMemberDB.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) + }) +} + func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) { return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0) diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 3a5f48d4c6..072429ed09 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -16,17 +16,19 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" ) @@ -108,6 +110,7 @@ type GroupDatabase interface { DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error @@ -115,6 +118,7 @@ type GroupDatabase interface { //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) + BatchFindMaxGroupMemberVersionCache(ctx context.Context, groupIDs []string) (map[string]*model.VersionLog, error) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) @@ -498,6 +502,29 @@ func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID strin return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit) } +func (g *groupDatabase) BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { + if len(groupIDs) == 0 { + return nil, errs.Wrap(errs.New("groupIDs is nil.")) + } + + // convert []uint64 to []uint + var uintVersions []uint + for _, version := range versions { + uintVersions = append(uintVersions, uint(version)) + } + + versionLogs, err := g.groupMemberDB.BatchFindMemberIncrVersion(ctx, groupIDs, uintVersions, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + groupMemberIncrVersionsMap := datautil.SliceToMap(versionLogs, func(e *model.VersionLog) string { + return e.DID + }) + + return groupMemberIncrVersionsMap, nil +} + func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) } @@ -506,6 +533,20 @@ func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, grou return g.cache.FindMaxGroupMemberVersion(ctx, groupID) } +func (g *groupDatabase) BatchFindMaxGroupMemberVersionCache(ctx context.Context, groupIDs []string) (map[string]*model.VersionLog, error) { + if len(groupIDs) == 0 { + return nil, errs.Wrap(errs.New("groupIDs is nil in Cache.")) + } + versionLogs, err := g.cache.BatchFindMaxGroupMemberVersion(ctx, groupIDs) + if err != nil { + return nil, errs.Wrap(err) + } + maxGroupMemberVersionsMap := datautil.SliceToMap(versionLogs, func(e *model.VersionLog) string { + return e.DID + }) + return maxGroupMemberVersionsMap, nil +} + func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { return g.cache.FindMaxJoinGroupVersion(ctx, userID) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index 43a7e6095e..0ddf0654c0 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -40,5 +41,6 @@ type GroupMember interface { JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint, limits []int) ([]*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 42b3dd72bd..2fdf2003b5 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/log" @@ -230,6 +231,11 @@ func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID stri return g.member.FindChangeLog(ctx, groupID, version, limit) } +func (g *GroupMemberMgo) BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint, limits []int) ([]*model.VersionLog, error) { + log.ZDebug(ctx, "Batch find member incr version", "groupIDs", groupIDs, "versions", versions) + return g.member.BatchFindChangeLog(ctx, groupIDs, versions, limits) +} + func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version) return g.join.FindChangeLog(ctx, userID, version, limit) diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 3b449007bc..2c4bdef4e8 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -3,6 +3,8 @@ package mgo import ( "context" "errors" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" @@ -13,7 +15,6 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" ) func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { @@ -35,6 +36,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { }, Options: options.Index().SetUnique(true), }) + return err } @@ -198,6 +200,26 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u } } +func (l *VersionLogMgo) BatchFindChangeLog(ctx context.Context, dIds []string, versions []uint, limits []int) (vLogs []*model.VersionLog, err error) { + for i := 0; i < len(dIds); i++ { + if vLog, err := l.findChangeLog(ctx, dIds[i], versions[i], limits[i]); err == nil { + vLogs = append(vLogs, vLog) + } else if !errors.Is(err, mongo.ErrNoDocuments) { + log.ZError(ctx, "findChangeLog error:", errs.Wrap(err)) + } + log.ZDebug(ctx, "init doc", "dId", dIds[i]) + if res, err := l.initDoc(ctx, dIds[i], nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dIds[i]) + vLogs = append(vLogs, res) + } else if mongo.IsDuplicateKeyError(err) { + l.findChangeLog(ctx, dIds[i], versions[i], limits[i]) + } else { + log.ZError(ctx, "init doc error:", errs.Wrap(err)) + } + } + return vLogs, errs.Wrap(err) +} + func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { if version == 0 && limit == 0 { return l.findDoc(ctx, dId) diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index 9d7bcc1724..28224a7c79 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -2,8 +2,9 @@ package database import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) const ( @@ -14,6 +15,7 @@ const ( type VersionLog interface { IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + BatchFindChangeLog(ctx context.Context, dIds []string, versions []uint, limits []int) ([]*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error Delete(ctx context.Context, dId string) error } From 8087f705c009d30311072c203371b3cba2869c5e Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:26:04 +0800 Subject: [PATCH 24/91] Fix search log (#2425) * feat: update alert email template * fix: search log * fix: log support ex --- config/email.tmpl | 44 +++++++++++++++++++-------- internal/rpc/third/log.go | 5 +-- pkg/common/storage/controller/user.go | 1 + 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/config/email.tmpl b/config/email.tmpl index 0385601d00..824144e9d7 100644 --- a/config/email.tmpl +++ b/config/email.tmpl @@ -1,16 +1,36 @@ {{ define "email.to.html" }} -{{ range .Alerts }} - -
-

OpenIM Alert

-

Alert Program: Prometheus Alert

-

Severity Level: {{ .Labels.severity }}

-

Alert Type: {{ .Labels.alertname }}

-

Affected Host: {{ .Labels.instance }}

-

Affected Service: {{ .Labels.job }}

-

Alert Subject: {{ .Annotations.summary }}

-

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

-
+{{ if eq .Status "firing" }} + {{ range .Alerts }} + +
+

OpenIM Alert

+

Alert Status: firing

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ {{ end }} + + +{{ else if eq .Status "resolved" }} + {{ range .Alerts }} + +
+

OpenIM Alert

+

Alert Status: resolved

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ {{ end }} {{ end }} {{ end }} diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 5c0b1f2e6b..cd52727cba 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -50,13 +50,14 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) platform := constant.PlatformID2Name[int(req.Platform)] for _, fileURL := range req.FileURLs { log := relationtb.Log{ - Version: req.Version, - SystemType: req.SystemType, Platform: platform, UserID: userID, CreateTime: time.Now(), Url: fileURL.URL, FileName: fileURL.Filename, + SystemType: req.SystemType, + Version: req.Version, + Ex: req.Ex, } for i := 0; i < 20; i++ { id := genLogID() diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 5ce8104e7b..533eac78f1 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -110,6 +110,7 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error // FindWithError Get the information of the specified user and return an error if the userID is not found. func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (users []*model.User, err error) { + userIDs = datautil.Distinct(userIDs) users, err = u.cache.GetUsersInfo(ctx, userIDs) if err != nil { return From 49ca5c998c96692bd61acdc9e9e69b24b6773994 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:20:42 +0800 Subject: [PATCH 25/91] feat: msg queue push (#2434) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/hub_server.go | 124 ++++++++++++++++++++---------- 3 files changed, 87 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 0637492eb5..fa40effddd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.38 - github.com/openimsdk/tools v0.0.49-alpha.51 + github.com/openimsdk/tools v0.0.49-alpha.52 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 92a783c069..cc0f5f7664 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.38 h1:kVZCHIXg/el8YJFoIBWhZu1sbbTUqmzgF4l0W3sUH24= github.com/openimsdk/protocol v0.0.69-alpha.38/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.51 h1:JTPEetVSNOczw1n+XjiPozaH2SBPQAc+9VlPE41wEeY= -github.com/openimsdk/tools v0.0.49-alpha.51/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/tools v0.0.49-alpha.52 h1:NwAAtBO4BV96qG6Z0P2btGEqn4AI2DFgaHvLMXNHal0= +github.com/openimsdk/tools v0.0.49-alpha.52/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 3891aa532c..28c227162a 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -22,11 +22,15 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mq/memamq" + "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" + "sync/atomic" ) func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { @@ -57,6 +61,7 @@ type Server struct { pushTerminal map[int]struct{} ready func(srv *Server) error userRcp rpcclient.UserRpcClient + queue *memamq.MemoryQueue } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { @@ -70,6 +75,7 @@ func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready f pushTerminal: make(map[int]struct{}), config: conf, ready: ready, + queue: memamq.NewMemoryQueue(512, 1024*16), } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} @@ -125,55 +131,93 @@ func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.Onli return nil, nil } -func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq, -) (*msggateway.OnlineBatchPushOneMsgResp, error) { - var singleUserResults []*msggateway.SingleMsgToUserResults - for _, v := range req.PushToUserIDs { - var resp []*msggateway.SingleMsgToUserPlatform - results := &msggateway.SingleMsgToUserResults{ - UserID: v, +func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { + clients, ok := s.LongConnServer.GetUserAllCons(userID) + if !ok { + log.ZDebug(ctx, "push user not online", "userID", userID) + return &msggateway.SingleMsgToUserResults{ + UserID: userID, } - clients, ok := s.LongConnServer.GetUserAllCons(v) - if !ok { - log.ZDebug(ctx, "push user not online", "userID", v) - results.Resp = resp - singleUserResults = append(singleUserResults, results) + } + log.ZDebug(ctx, "push user online", "clients", clients, "userID", userID) + result := &msggateway.SingleMsgToUserResults{ + UserID: userID, + Resp: make([]*msggateway.SingleMsgToUserPlatform, 0, len(clients)), + } + for _, client := range clients { + if client == nil { continue } - - log.ZDebug(ctx, "push user online", "clients", clients, "userID", v) - for _, client := range clients { - if client == nil { - continue + userPlatform := &msggateway.SingleMsgToUserPlatform{ + RecvPlatFormID: int32(client.PlatformID), + } + if !client.IsBackground || + (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { + err := client.PushMessage(ctx, msgData) + if err != nil { + userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) + } else { + if _, ok := s.pushTerminal[client.PlatformID]; ok { + result.OnlinePush = true + } } + } else { + userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) + } + result.Resp = append(result.Resp, userPlatform) + } + return result +} - userPlatform := &msggateway.SingleMsgToUserPlatform{ - RecvPlatFormID: int32(client.PlatformID), +func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { + if len(req.PushToUserIDs) == 0 { + return &msggateway.OnlineBatchPushOneMsgResp{}, nil + } + ch := make(chan *msggateway.SingleMsgToUserResults, len(req.PushToUserIDs)) + var count atomic.Int64 + count.Add(int64(len(req.PushToUserIDs))) + for i := range req.PushToUserIDs { + userID := req.PushToUserIDs[i] + err := s.queue.PushCtx(ctx, func() { + ch <- s.pushToUser(ctx, userID, req.MsgData) + if count.Add(-1) == 0 { + close(ch) } - if !client.IsBackground || - (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { - err := client.PushMessage(ctx, req.MsgData) - if err != nil { - userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) - resp = append(resp, userPlatform) - } else { - if _, ok := s.pushTerminal[client.PlatformID]; ok { - results.OnlinePush = true - resp = append(resp, userPlatform) - } - } - } else { - userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) - resp = append(resp, userPlatform) + }) + if err != nil { + if count.Add(-1) == 0 { + close(ch) + } + log.ZError(ctx, "pushToUser MemoryQueue failed", err, "userID", userID) + ch <- &msggateway.SingleMsgToUserResults{ + UserID: userID, } } - results.Resp = resp - singleUserResults = append(singleUserResults, results) } - - return &msggateway.OnlineBatchPushOneMsgResp{ - SinglePushResult: singleUserResults, - }, nil + resp := &msggateway.OnlineBatchPushOneMsgResp{ + SinglePushResult: make([]*msggateway.SingleMsgToUserResults, 0, len(req.PushToUserIDs)), + } + for { + select { + case <-ctx.Done(): + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ctx done", context.Cause(ctx)) + userIDSet := datautil.SliceSet(req.PushToUserIDs) + for _, results := range resp.SinglePushResult { + delete(userIDSet, results.UserID) + } + for userID := range userIDSet { + resp.SinglePushResult = append(resp.SinglePushResult, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return resp, nil + case res, ok := <-ch: + if !ok { + return resp, nil + } + resp.SinglePushResult = append(resp.SinglePushResult, res) + } + } } func (s *Server) KickUserOffline( From aab9c79e0650bbb69d686a35fbf97468852c3db3 Mon Sep 17 00:00:00 2001 From: totalo Date: Wed, 24 Jul 2024 14:50:02 +0800 Subject: [PATCH 26/91] docs: fix english typo (#2432) --- README.md | 2 +- README_zh_CN.md | 2 +- docs/contrib/util-scripts.md | 2 +- docs/contributing/CONTRIBUTING-JP.md | 2 +- docs/contributing/CONTRIBUTING-PL.md | 2 +- docs/readme/README_cs.md | 2 +- docs/readme/README_da.md | 2 +- docs/readme/README_el.md | 2 +- docs/readme/README_es.md | 2 +- docs/readme/README_fa.md | 2 +- docs/readme/README_fr.md | 2 +- docs/readme/README_hu.md | 2 +- docs/readme/README_ja.md | 2 +- docs/readme/README_ko.md | 2 +- docs/readme/README_tr.md | 2 +- docs/readme/README_uk.md | 2 +- docs/readme/README_vi.md | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d73d5749a5..a99559cdb0 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/README_zh_CN.md b/README_zh_CN.md index 65aac9ebc6..59198eafb6 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/contrib/util-scripts.md b/docs/contrib/util-scripts.md index 0bf6f23e5c..30da871a49 100644 --- a/docs/contrib/util-scripts.md +++ b/docs/contrib/util-scripts.md @@ -32,7 +32,7 @@ This script offers a variety of utilities and helpers to enhance and simplify op ## brief descriptions of each function -**Englist:** +**English:** 1. `openim::util::ensure-gnu-sed` - Determines if GNU version of `sed` exists on the system and sets its name. 2. `openim::util::ensure-gnu-date` - Determines if GNU version of `date` exists on the system and sets its name. 3. `openim::util::check-file-in-alphabetical-order` - Checks if a file is sorted in alphabetical order. diff --git a/docs/contributing/CONTRIBUTING-JP.md b/docs/contributing/CONTRIBUTING-JP.md index 86bbfefcd7..1798d4e3d4 100644 --- a/docs/contributing/CONTRIBUTING-JP.md +++ b/docs/contributing/CONTRIBUTING-JP.md @@ -1,7 +1,7 @@ # How do I contribute code to OpenIM

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/contributing/CONTRIBUTING-PL.md b/docs/contributing/CONTRIBUTING-PL.md index 86bbfefcd7..1798d4e3d4 100644 --- a/docs/contributing/CONTRIBUTING-PL.md +++ b/docs/contributing/CONTRIBUTING-PL.md @@ -1,7 +1,7 @@ # How do I contribute code to OpenIM

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_cs.md b/docs/readme/README_cs.md index 5a9eeb2324..63f730a516 100644 --- a/docs/readme/README_cs.md +++ b/docs/readme/README_cs.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_da.md b/docs/readme/README_da.md index 1b776ddb8b..60d97348a4 100644 --- a/docs/readme/README_da.md +++ b/docs/readme/README_da.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index 252521f351..da01fcb470 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index cd1b7290ee..f123b85c36 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index 49f05cd4c3..7a1512e844 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index e707fc59b7..aaf7a9bd41 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_hu.md b/docs/readme/README_hu.md index 57f0066928..61013c3343 100644 --- a/docs/readme/README_hu.md +++ b/docs/readme/README_hu.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index bd94b11538..5a083c1bf8 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index bd7a1aed37..ebcdd71eee 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index ca2a816db0..3cf19f537e 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 30bc767305..81820590be 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index e500da6d2d..a6ab39253b 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -19,7 +19,7 @@

- Englist · + English · 中文 · Українська · Česky · From 88ad85b5853ddc89efb078ca6e1b8e82d0537abf Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:06:26 +0800 Subject: [PATCH 27/91] fix: batchGetMaxSeq bug (#2438) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- .../storage/cache/redis/seq_conversation.go | 4 ++-- pkg/common/storage/database/mgo/msg.go | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index fb8a547dfa..7fe849193e 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -77,8 +77,8 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s return errs.Wrap(err) } } - if len(notFoundKey) > 0 { - conversationID := keyConversationID[notFoundKey[0]] + for _, key := range notFoundKey { + conversationID := keyConversationID[key] seq, err := s.GetMaxSeq(ctx, conversationID) if err != nil { return err diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index ebabadf52c..2d1819c3a6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -94,6 +94,29 @@ func (m *MsgMgo) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDo } func (m *MsgMgo) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { + msgs, err := m.getMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs) + if err != nil { + return nil, err + } + if len(msgs) == len(seqs) { + return msgs, nil + } + tmp := make(map[int64]*model.MsgInfoModel) + for i, val := range msgs { + tmp[val.Msg.Seq] = msgs[i] + } + res := make([]*model.MsgInfoModel, 0, len(seqs)) + for _, seq := range seqs { + if val, ok := tmp[seq]; ok { + res = append(res, val) + } else { + res = append(res, &model.MsgInfoModel{Msg: &model.MsgDataModel{Seq: seq}}) + } + } + return res, nil +} + +func (m *MsgMgo) getMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { indexs := make([]int64, 0, len(seqs)) for _, seq := range seqs { indexs = append(indexs, m.model.GetMsgIndex(seq)) From 80b332cb8227997595ff69da5a5db234f9fab8e0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 24 Jul 2024 19:27:02 +0800 Subject: [PATCH 28/91] feat: implement log isSimpilfy. (#2436) * feat: implement log isSimpilfy. * update go mod. --- config/log.yml | 3 ++- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/common/cmd/root.go | 1 + pkg/common/config/config.go | 6 ++++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/config/log.yml b/config/log.yml index 2194d89174..8620af611b 100644 --- a/config/log.yml +++ b/config/log.yml @@ -10,4 +10,5 @@ remainLogLevel: 6 isStdout: false # Whether to log in JSON format, default is acceptable isJson: false - +# output simplify log when KeyAndValues's value len is bigger than 50 in rpc method log +isSimplify: true \ No newline at end of file diff --git a/go.mod b/go.mod index fa40effddd..fe3f8afbc2 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.38 - github.com/openimsdk/tools v0.0.49-alpha.52 + github.com/openimsdk/protocol v0.0.69-alpha.41 + github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index cc0f5f7664..e59418cd80 100644 --- a/go.sum +++ b/go.sum @@ -319,10 +319,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.38 h1:kVZCHIXg/el8YJFoIBWhZu1sbbTUqmzgF4l0W3sUH24= -github.com/openimsdk/protocol v0.0.69-alpha.38/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.52 h1:NwAAtBO4BV96qG6Z0P2btGEqn4AI2DFgaHvLMXNHal0= -github.com/openimsdk/tools v0.0.49-alpha.52/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/protocol v0.0.69-alpha.41 h1:9hoQ6UHMBq+g58KXir90EpnnvwJ1bvDPixPSaODo4nY= +github.com/openimsdk/protocol v0.0.69-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= +github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 08bb6d0642..84e9856973 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -139,6 +139,7 @@ func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { r.log.RemainRotationCount, r.log.RotationTime, config.Version, + r.log.IsSimplify, ) if err != nil { return errs.Wrap(err) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index f018c7b834..c6c672eb8d 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,6 +15,9 @@ package config import ( + "strings" + "time" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" @@ -22,8 +25,6 @@ import ( "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "strings" - "time" ) type CacheConfig struct { @@ -48,6 +49,7 @@ type Log struct { RemainLogLevel int `mapstructure:"remainLogLevel"` IsStdout bool `mapstructure:"isStdout"` IsJson bool `mapstructure:"isJson"` + IsSimplify bool `mapstructure:"isSimplify"` WithStack bool `mapstructure:"withStack"` } From ebdc91a966c1cf028ad9cbadbe44fb72e961e7b3 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:01:33 +0800 Subject: [PATCH 29/91] fix: user seq bug (#2442) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- pkg/common/storage/cache/redis/batch.go | 2 + .../storage/cache/redis/batch_handler.go | 1 + pkg/common/storage/cache/redis/seq_user.go | 40 +++++++++---------- pkg/common/storage/cache/seq_user.go | 18 ++++----- pkg/common/storage/controller/msg.go | 20 +++++----- pkg/common/storage/database/mgo/seq_user.go | 14 +++---- pkg/common/storage/database/seq_user.go | 14 +++---- tools/seq/internal/main.go | 8 ++-- 8 files changed, 60 insertions(+), 57 deletions(-) diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 5f9a8c82db..4d65c59298 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "golang.org/x/sync/singleflight" "time" @@ -49,6 +50,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } values, err := fn(ctx, queryIds) if err != nil { + log.ZError(ctx, "batchGetCache query database failed", err, "keys", keys, "queryIds", queryIds) return nil, err } if len(values) == 0 { diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 52e046a408..f9923e198e 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -118,6 +118,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { + log.ZError(ctx, "getCache query database failed", err, "key", key) return "", err } bs, err := json.Marshal(t) diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index 2ad43eebdb..edbc66b21b 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -44,38 +44,38 @@ func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID s return cachekey.GetSeqUserReadSeqKey(conversationID, userID) } -func (s *seqUserCacheRedis) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserMaxSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - if err := s.mgo.SetMaxSeq(ctx, conversationID, userID, seq); err != nil { +func (s *seqUserCacheRedis) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetUserMaxSeq(ctx, conversationID, userID, seq); err != nil { return err } return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) } -func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserMinSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - return s.SetMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) +func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) } -func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { - return s.mgo.GetMaxSeq(ctx, conversationID, userID) + return s.mgo.GetUserReadSeq(ctx, conversationID, userID) }) } -func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -85,10 +85,10 @@ func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID strin return nil } -func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { keys := make([]string, 0, len(seqs)) for conversationID, seq := range seqs { - if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserMinSeq(ctx, conversationID, userID, seq); err != nil { return err } keys = append(keys, s.getSeqUserMinSeqKey(conversationID, userID)) @@ -96,7 +96,7 @@ func (s *seqUserCacheRedis) SetMinSeqs(ctx context.Context, userID string, seqs return DeleteCacheBySlot(ctx, s.rocks, keys) } -func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { keys := make([]string, 0, len(seqs)) keySeq := make(map[string]int64) for conversationID, seq := range seqs { @@ -121,16 +121,16 @@ func (s *seqUserCacheRedis) setRedisReadSeqs(ctx context.Context, userID string, return nil } -func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { +func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { if len(seqs) == 0 { return nil } - if err := s.setRedisReadSeqs(ctx, userID, seqs); err != nil { + if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil { return err } for conversationID, seq := range seqs { if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -138,13 +138,13 @@ func (s *seqUserCacheRedis) SetReadSeqs(ctx context.Context, userID string, seqs return nil } -func (s *seqUserCacheRedis) GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { +func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { return s.getSeqUserReadSeqKey(conversationID, userID) }, func(v *readSeqModel) string { return v.ConversationID }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { - seqs, err := s.mgo.GetReadSeqs(ctx, userID, conversationIDs) + seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs) if err != nil { return nil, err } diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go index 4d0bb4ffab..61dbc0ab45 100644 --- a/pkg/common/storage/cache/seq_user.go +++ b/pkg/common/storage/cache/seq_user.go @@ -3,13 +3,13 @@ package cache import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error - SetMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - SetReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error - GetReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error + SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 32202ac9e9..a6f0dbbb04 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -334,7 +334,7 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } @@ -498,7 +498,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -576,7 +576,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -674,12 +674,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -794,23 +794,23 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int } func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seqUser.SetMinSeqs(ctx, userID, seqs) + return db.seqUser.SetUserMinSeqs(ctx, userID, seqs) } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seqUser.SetReadSeqs(ctx, userID, hasReadSeqs) + return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetReadSeq(ctx, conversationID, userID, hasReadSeq) + return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seqUser.GetReadSeqs(ctx, userID, conversationIDs) + return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seqUser.GetReadSeq(ctx, conversationID, userID) + return db.seqUser.GetUserReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index e0cbb08d9c..7c9a8f1336 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -68,27 +68,27 @@ func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID } } -func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "max_seq") } -func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "max_seq") } -func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "min_seq") } -func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "min_seq") } -func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { +func (s *seqUserMongo) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return s.getSeq(ctx, conversationID, userID, "read_seq") } -func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { +func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { if len(conversationID) == 0 { return map[string]int64{}, nil } @@ -105,6 +105,6 @@ func (s *seqUserMongo) GetReadSeqs(ctx context.Context, userID string, conversat return res, nil } -func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { +func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go index edd3910d02..9f75c710b6 100644 --- a/pkg/common/storage/database/seq_user.go +++ b/pkg/common/storage/database/seq_user.go @@ -3,11 +3,11 @@ package database import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) - SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error - GetReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) } diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 7f021a90e8..2bec5a8f1c 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -130,14 +130,14 @@ func Main(conf string, del time.Duration) error { if err != nil { return 0, err } - return uSeq.GetReadSeq(ctx, conversationID, userID) + return uSeq.GetUserReadSeq(ctx, conversationID, userID) }, SetSeq: func(ctx context.Context, id string, seq int64) error { conversationID, userID, err := uSpitHasReadSeq(id) if err != nil { return err } - return uSeq.SetReadSeq(ctx, conversationID, userID, seq) + return uSeq.SetUserReadSeq(ctx, conversationID, userID, seq) }, }, { @@ -147,14 +147,14 @@ func Main(conf string, del time.Duration) error { if err != nil { return 0, err } - return uSeq.GetMinSeq(ctx, conversationID, userID) + return uSeq.GetUserMinSeq(ctx, conversationID, userID) }, SetSeq: func(ctx context.Context, id string, seq int64) error { conversationID, userID, err := uSpitConversationUserMinSeq(id) if err != nil { return err } - return uSeq.SetMinSeq(ctx, conversationID, userID, seq) + return uSeq.SetUserMinSeq(ctx, conversationID, userID, seq) }, }, } From a8b84911a42bab22d0b38eb0cf47ba6606f0e98f Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:40:36 +0800 Subject: [PATCH 30/91] fix: display is read (#2444) --- internal/rpc/group/notification.go | 45 ++---------------------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index a7398795fa..9815167e9b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -17,6 +17,7 @@ package group import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" @@ -126,25 +127,8 @@ func (g *GroupNotificationSender) getGroupInfo(ctx context.Context, groupID stri if len(ownerUserIDs) > 0 { ownerUserID = ownerUserIDs[0] } - return &sdkws.GroupInfo{ - GroupID: gm.GroupID, - GroupName: gm.GroupName, - Notification: gm.Notification, - Introduction: gm.Introduction, - FaceURL: gm.FaceURL, - OwnerUserID: ownerUserID, - CreateTime: gm.CreateTime.UnixMilli(), - MemberCount: num, - Ex: gm.Ex, - Status: gm.Status, - CreatorUserID: gm.CreatorUserID, - GroupType: gm.GroupType, - NeedVerification: gm.NeedVerification, - LookMemberInfo: gm.LookMemberInfo, - ApplyMemberFriend: gm.ApplyMemberFriend, - NotificationUpdateTime: gm.NotificationUpdateTime.UnixMilli(), - NotificationUserID: gm.NotificationUserID, - }, nil + + return convert.Db2PbGroupInfo(gm, ownerUserID, num), nil } func (g *GroupNotificationSender) getGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { @@ -198,29 +182,6 @@ func (g *GroupNotificationSender) getGroupOwnerAndAdminUserID(ctx context.Contex return datautil.Slice(members, fn), nil } -//nolint:unused -func (g *GroupNotificationSender) groupDB2PB(group *model.Group, ownerUserID string, memberCount uint32) *sdkws.GroupInfo { - return &sdkws.GroupInfo{ - GroupID: group.GroupID, - GroupName: group.GroupName, - Notification: group.Notification, - Introduction: group.Introduction, - FaceURL: group.FaceURL, - OwnerUserID: ownerUserID, - CreateTime: group.CreateTime.UnixMilli(), - MemberCount: memberCount, - Ex: group.Ex, - Status: group.Status, - CreatorUserID: group.CreatorUserID, - GroupType: group.GroupType, - NeedVerification: group.NeedVerification, - LookMemberInfo: group.LookMemberInfo, - ApplyMemberFriend: group.ApplyMemberFriend, - NotificationUpdateTime: group.NotificationUpdateTime.UnixMilli(), - NotificationUserID: group.NotificationUserID, - } -} - func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { return &sdkws.GroupMemberFullInfo{ GroupID: member.GroupID, From c3c9969f2f15f182f280f303aa5cc870ca56bb99 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 26 Jul 2024 15:52:37 +0800 Subject: [PATCH 31/91] chore: add debug log in writePongMsg (#2446) * update protocol in go mod. * add debug log in writePongMsg. * update log level. --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 12 +++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index fe3f8afbc2..71301d2909 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.41 + github.com/openimsdk/protocol v0.0.69-alpha.42 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index e59418cd80..53060b1984 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.41 h1:9hoQ6UHMBq+g58KXir90EpnnvwJ1bvDPixPSaODo4nY= -github.com/openimsdk/protocol v0.0.69-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.42 h1:Vwuru2NtyTHuqaM+1JGxcoGvP25QWjS92oI0zGJp+lM= +github.com/openimsdk/protocol v0.0.69-alpha.42/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 889e5c456e..ded830c43a 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -107,7 +107,6 @@ func (c *Client) pingHandler(appData string) error { } log.ZDebug(c.ctx, "ping Handler Success.", "appData", appData) - return c.writePongMsg(appData) } @@ -392,17 +391,24 @@ func (c *Client) writePingMsg() error { } func (c *Client) writePongMsg(appData string) error { + log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) if c.closed.Load() { return nil } + log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) c.w.Lock() defer c.w.Unlock() + log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) err := c.conn.SetWriteDeadline(writeWait) if err != nil { - return err + return errs.Wrap(err) + } + err = c.conn.WriteMessage(PongMessage, []byte(appData)) + if err != nil { + log.ZWarn(c.ctx, "Write Message have error", errs.Wrap(err), "Pong msg", PongMessage) } - return c.conn.WriteMessage(PongMessage, []byte(appData)) + return errs.Wrap(err) } From 5c52840d670e1a280335015d648bc090577cb068 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:53:25 +0800 Subject: [PATCH 32/91] fix: user seq, asynchronous friend notification, message search (#2447) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- internal/rpc/friend/friend.go | 4 +- internal/rpc/friend/sync.go | 22 +- internal/rpc/msg/sync_msg.go | 4 +- pkg/common/storage/cache/redis/batch_test.go | 2 +- .../storage/cache/redis/seq_user_test.go | 32 ++ pkg/common/storage/controller/msg.go | 4 +- pkg/common/storage/database/mgo/msg.go | 457 ++++++++++++++---- pkg/common/storage/database/mgo/msg_test.go | 75 +++ .../database/mgo/seq_conversation_test.go | 7 +- pkg/common/storage/database/mgo/seq_user.go | 9 + pkg/common/storage/database/msg.go | 2 +- 11 files changed, 518 insertions(+), 100 deletions(-) create mode 100644 pkg/common/storage/database/mgo/msg_test.go diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 622e19f422..bdb786bca4 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -49,6 +50,7 @@ type friendServer struct { RegisterCenter discovery.SvcDiscoveryRegistry config *Config webhookClient *webhook.Client + queue *memamq.MemoryQueue } type Config struct { @@ -118,8 +120,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + queue: memamq.NewMemoryQueue(128, 1024*8), }) - return nil } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 145c287da0..902cc73037 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" "slices" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" @@ -17,14 +18,23 @@ func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *rela if err != nil { return nil, err } - for _, userID := range userIDs { - if err := s.db.OwnerIncrVersion(ctx, userID, []string{req.UserID}, model.VersionStateUpdate); err != nil { - return nil, err + if len(userIDs) > 0 { + friendUserIDs := []string{req.UserID} + noCancelCtx := context.WithoutCancel(ctx) + err := s.queue.PushCtx(ctx, func() { + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(noCancelCtx, userID, friendUserIDs, model.VersionStateUpdate); err != nil { + log.ZError(ctx, "OwnerIncrVersion", err, "userID", userID, "friendUserIDs", friendUserIDs) + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(noCancelCtx, req.UserID, userID) + } + }) + if err != nil { + log.ZError(ctx, "NotificationUserInfoUpdate timeout", err, "userID", req.UserID) } } - for _, userID := range userIDs { - s.notificationSender.FriendInfoUpdatedNotification(ctx, req.UserID, userID) - } return &relation.NotificationUserInfoUpdateResp{}, nil } diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index afb79506ee..f5b5ebda53 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -111,7 +111,7 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (resp *msg.SearchMessageResp, err error) { var chatLogs []*sdkws.MsgData - var total int32 + var total int64 resp = &msg.SearchMessageResp{} if total, chatLogs, err = m.MsgDatabase.SearchMessage(ctx, req); err != nil { return nil, err @@ -194,7 +194,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq } resp.ChatLogs = append(resp.ChatLogs, pbchatLog) } - resp.ChatLogsNum = total + resp.ChatLogsNum = int32(total) return resp, nil } diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go index e4caa2a21f..bbb6d76f15 100644 --- a/pkg/common/storage/cache/redis/batch_test.go +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -45,7 +45,7 @@ func TestName(t *testing.T) { } seqUser := NewSeqUserCacheRedis(rdb, mgoSeqUser) - res, err := seqUser.GetReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) + res, err := seqUser.GetUserReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) if err != nil { panic(err) } diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index e4fd95922a..0059c81db9 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + mgo2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "log" "strconv" "sync/atomic" @@ -77,3 +80,32 @@ func TestRecvOnline(t *testing.T) { fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) } } + +func TestName1(t *testing.T) { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 0, + } + rdb := redis.NewClient(opt) + + mgo, err := mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo2.NewSeqUserMongo(mgo.Database("openim_v3")) + if err != nil { + panic(err) + } + seq := NewSeqUserCacheRedis(rdb, model) + + res, err := seq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"}) + if err != nil { + panic(err) + } + t.Log(res) + +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a6f0dbbb04..4ea74ef690 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -84,7 +84,7 @@ type CommonMsgDatabase interface { //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) + SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) FindOneByDocIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) // to mq @@ -878,7 +878,7 @@ func (db *commonMsgDatabase) RangeGroupSendCount( return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) { +func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) { var totalMsgs []*sdkws.MsgData total, msgs, err := db.msgDocDatabase.SearchMessage(ctx, req) if err != nil { diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 2d1819c3a6..03f47c5033 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -278,124 +278,409 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do return nil } -func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { - where := make(bson.A, 0, 6) +//func (m *MsgMgo) searchCount(ctx context.Context, filter any) (int64, error) { +// +// return nil, nil +//} + +//func (m *MsgMgo) searchMessage(ctx context.Context, filter any, nextID primitive.ObjectID, content bool, limit int) (int64, []*model.MsgInfoModel, primitive.ObjectID, error) { +// var pipeline bson.A +// if !nextID.IsZero() { +// pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) +// } +// pipeline = append(pipeline, +// bson.M{"$match": filter}, +// bson.M{"$limit": limit}, +// bson.M{"$unwind": "$msgs"}, +// bson.M{"$match": filter}, +// bson.M{ +// "$group": bson.M{ +// "_id": "$_id", +// "doc_id": bson.M{ +// "$first": "$doc_id", +// }, +// "msgs": bson.M{"$push": "$msgs"}, +// }, +// }, +// ) +// if !content { +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 1, +// "count": bson.M{"$size": "$msgs"}, +// }, +// }, +// ) +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Count int64 `bson:"count"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, nil +// } +// var count int64 +// for _, r := range res { +// count += r.Count +// } +// return count, nil, res[len(res)-1].ID, nil +// } +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Msg []*model.MsgInfoModel `bson:"msgs"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, err +// } +// var count int +// for _, r := range res { +// count += len(r.Msg) +// } +// msgs := make([]*model.MsgInfoModel, 0, count) +// for _, r := range res { +// msgs = append(msgs, r.Msg...) +// } +// return int64(count), msgs, res[len(res)-1].ID, nil +//} + +/* + +db.msg3.aggregate( + [ + { + "$match": { + "doc_id": "si_7009965934_8710838466:0" + }, + + } + ] +) + + +*/ + +type searchMessageIndex struct { + ID primitive.ObjectID `bson:"_id"` + Index []int64 `bson:"index"` +} + +func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID primitive.ObjectID, limit int) ([]searchMessageIndex, error) { + var pipeline bson.A + if !nextID.IsZero() { + pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) + } + pipeline = append(pipeline, + bson.M{"$sort": bson.M{"_id": 1}}, + bson.M{"$match": filter}, + bson.M{"$limit": limit}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs": bson.M{ + "$map": bson.M{ + "input": "$msgs", + "as": "msg", + "in": bson.M{ + "$mergeObjects": bson.A{ + "$$msg", + bson.M{ + "_search_temp_index": bson.M{ + "$indexOfArray": bson.A{ + "$msgs", "$$msg", + }, + }, + }, + }, + }, + }, + }, + }, + }, + bson.M{"$unwind": "$msgs"}, + bson.M{"$match": filter}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs._search_temp_index": 1, + }, + }, + bson.M{ + "$group": bson.M{ + "_id": "$_id", + "index": bson.M{"$push": "$msgs._search_temp_index"}, + }, + }, + bson.M{"$sort": bson.M{"_id": 1}}, + ) + return mongoutil.Aggregate[searchMessageIndex](ctx, m.coll, pipeline) +} + +func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []searchMessageIndex, error) { + filter := bson.M{} if req.RecvID != "" { - where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) + filter["$or"] = bson.A{ + bson.M{"msgs.msg.recv_id": req.RecvID}, + bson.M{"msgs.msg.group_id": req.RecvID}, + } } if req.SendID != "" { - where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) + filter["msgs.msg.send_id"] = req.SendID } if req.ContentType != 0 { - where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) + filter["msgs.msg.content_type"] = req.ContentType } if req.SessionType != 0 { - where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) + filter["msgs.msg.session_type"] = req.SessionType } if req.SendTime != "" { sendTime, err := time.Parse(time.DateOnly, req.SendTime) if err != nil { return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) } - where = append(where, - bson.M{ - "msgs.msg.send_time": bson.M{ - "$gte": sendTime.UnixMilli(), - }, - }, + filter["$and"] = bson.A{ + bson.M{"msgs.msg.send_time": bson.M{ + "$gte": sendTime.UnixMilli(), + }}, bson.M{ "msgs.msg.send_time": bson.M{ "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), }, }, - ) - } - pipeline := bson.A{ - bson.M{ - "$unwind": "$msgs", - }, - } - if len(where) > 0 { - pipeline = append(pipeline, bson.M{ - "$match": bson.M{"$and": where}, - }) + } } - pipeline = append(pipeline, - bson.M{ - "$project": bson.M{ - "_id": 0, - "msg": "$msgs.msg", - }, - }, - bson.M{ - "$count": "count", - }, + + var ( + nextID primitive.ObjectID + count int + dataRange []searchMessageIndex + skip = int((req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber()) ) - count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) - if err != nil { - return 0, nil, err + _, _ = dataRange, skip + const maxDoc = 50 + data := make([]searchMessageIndex, 0, req.Pagination.GetShowNumber()) + push := cap(data) + for i := 0; ; i++ { + res, err := m.searchMessageIndex(ctx, filter, nextID, maxDoc) + if err != nil { + return 0, nil, err + } + if len(res) > 0 { + nextID = res[len(res)-1].ID + } + for _, r := range res { + var dataIndex []int64 + for _, index := range r.Index { + if push > 0 && count >= skip { + dataIndex = append(dataIndex, index) + push-- + } + count++ + } + if len(dataIndex) > 0 { + data = append(data, searchMessageIndex{ + ID: r.ID, + Index: dataIndex, + }) + } + } + if push <= 0 { + push-- + } + if len(res) < maxDoc || push < -10 { + return int64(count), data, nil + } + } +} + +func (m *MsgMgo) getDocRange(ctx context.Context, id primitive.ObjectID, index []int64) ([]*model.MsgInfoModel, error) { + if len(index) == 0 { + return nil, nil } - if len(count) == 0 || count[0] == 0 { - return 0, nil, nil + + pipeline := bson.A{ + bson.M{"$match": bson.M{"_id": id}}, + bson.M{"$project": "$msgs"}, } - pipeline = pipeline[:len(pipeline)-1] - pipeline = append(pipeline, - bson.M{ - "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), - }, - bson.M{ - "$limit": req.Pagination.GetShowNumber(), - }, - ) msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + return msgs, nil +} + +func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) { + count, data, err := m.searchMessage(ctx, req) if err != nil { return 0, nil, err } - for i := range msgs { - msgInfo := msgs[i] - if msgInfo == nil || msgInfo.Msg == nil { - continue + var msgs []*model.MsgInfoModel + if len(data) > 0 { + var n int + for _, d := range data { + n += len(d.Index) } - if msgInfo.Revoke != nil { - revokeContent := sdkws.MessageRevokedContent{ - RevokerID: msgInfo.Revoke.UserID, - RevokerRole: msgInfo.Revoke.Role, - ClientMsgID: msgInfo.Msg.ClientMsgID, - RevokerNickname: msgInfo.Revoke.Nickname, - RevokeTime: msgInfo.Revoke.Time, - SourceMessageSendTime: msgInfo.Msg.SendTime, - SourceMessageSendID: msgInfo.Msg.SendID, - SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, - SessionType: msgInfo.Msg.SessionType, - Seq: msgInfo.Msg.Seq, - Ex: msgInfo.Msg.Ex, - } - data, err := jsonutil.JsonMarshal(&revokeContent) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") - } - elem := sdkws.NotificationElem{Detail: string(data)} - content, err := jsonutil.JsonMarshal(&elem) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal elem") + msgs = make([]*model.MsgInfoModel, 0, n) + } + for _, val := range data { + res, err := mongoutil.FindOne[*model.MsgDocModel](ctx, m.coll, bson.M{"_id": val.ID}) + if err != nil { + return 0, nil, err + } + for _, i := range val.Index { + if i >= int64(len(res.Msg)) { + continue } - msgInfo.Msg.ContentType = constant.MsgRevokeNotification - msgInfo.Msg.Content = string(content) + msgs = append(msgs, res.Msg[i]) } } - //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber - //n := int32(len(msgs)) - //if start >= n { - // return n, []*relation.MsgInfoModel{}, nil - //} - //if start+req.Pagination.ShowNumber < n { - // msgs = msgs[start : start+req.Pagination.ShowNumber] - //} else { - // msgs = msgs[start:] - //} - return count[0], msgs, nil + return count, msgs, nil } +//func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { +// where := make(bson.A, 0, 6) +// if req.RecvID != "" { +// if req.SessionType == constant.ReadGroupChatType { +// where = append(where, bson.M{ +// "$or": bson.A{ +// bson.M{"doc_id": "^n_" + req.RecvID + ":"}, +// bson.M{"doc_id": "^sg_" + req.RecvID + ":"}, +// }, +// }) +// } else { +// where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) +// } +// } +// if req.SendID != "" { +// where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) +// } +// if req.ContentType != 0 { +// where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) +// } +// if req.SessionType != 0 { +// where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) +// } +// if req.SendTime != "" { +// sendTime, err := time.Parse(time.DateOnly, req.SendTime) +// if err != nil { +// return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) +// } +// where = append(where, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$gte": sendTime.UnixMilli(), +// }, +// }, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), +// }, +// }, +// ) +// } +// opt := options.Find().SetLimit(100) +// res, err := mongoutil.Find[model.MsgDocModel](ctx, m.coll, bson.M{"$and": where}, opt) +// if err != nil { +// return 0, nil, err +// } +// _ = res +// fmt.Println() +// +// return 0, nil, nil +// pipeline := bson.A{ +// bson.M{ +// "$unwind": "$msgs", +// }, +// } +// if len(where) > 0 { +// pipeline = append(pipeline, bson.M{ +// "$match": bson.M{"$and": where}, +// }) +// } +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 0, +// "msg": "$msgs.msg", +// }, +// }, +// bson.M{ +// "$count": "count", +// }, +// ) +// //count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) +// //if err != nil { +// // return 0, nil, err +// //} +// //if len(count) == 0 || count[0] == 0 { +// // return 0, nil, nil +// //} +// count := []int32{0} +// pipeline = pipeline[:len(pipeline)-1] +// pipeline = append(pipeline, +// bson.M{ +// "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), +// }, +// bson.M{ +// "$limit": req.Pagination.GetShowNumber(), +// }, +// ) +// msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, err +// } +// for i := range msgs { +// msgInfo := msgs[i] +// if msgInfo == nil || msgInfo.Msg == nil { +// continue +// } +// if msgInfo.Revoke != nil { +// revokeContent := sdkws.MessageRevokedContent{ +// RevokerID: msgInfo.Revoke.UserID, +// RevokerRole: msgInfo.Revoke.Role, +// ClientMsgID: msgInfo.Msg.ClientMsgID, +// RevokerNickname: msgInfo.Revoke.Nickname, +// RevokeTime: msgInfo.Revoke.Time, +// SourceMessageSendTime: msgInfo.Msg.SendTime, +// SourceMessageSendID: msgInfo.Msg.SendID, +// SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, +// SessionType: msgInfo.Msg.SessionType, +// Seq: msgInfo.Msg.Seq, +// Ex: msgInfo.Msg.Ex, +// } +// data, err := jsonutil.JsonMarshal(&revokeContent) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") +// } +// elem := sdkws.NotificationElem{Detail: string(data)} +// content, err := jsonutil.JsonMarshal(&elem) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal elem") +// } +// msgInfo.Msg.ContentType = constant.MsgRevokeNotification +// msgInfo.Msg.Content = string(content) +// } +// } +// //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber +// //n := int32(len(msgs)) +// //if start >= n { +// // return n, []*relation.MsgInfoModel{}, nil +// //} +// //if start+req.Pagination.ShowNumber < n { +// // msgs = msgs[start : start+req.Pagination.ShowNumber] +// //} else { +// // msgs = msgs[start:] +// //} +// return count[0], msgs, nil +//} + func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { var sort int if ase { diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go new file mode 100644 index 0000000000..5aed4dc511 --- /dev/null +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -0,0 +1,75 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "math/rand" + "strconv" + "testing" + "time" +) + +func TestName1(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + + req := &msg.SearchMessageReq{ + //RecvID: "3187706596", + //SendID: "7009965934", + ContentType: 101, + //SendTime: "2024-05-06", + //SessionType: 3, + Pagination: &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 10, + }, + } + total, res, err := v.SearchMessage(ctx, req) + if err != nil { + panic(err) + } + + for i, re := range res { + t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + } + + t.Log(total) +} + +func TestName10(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + opt := options.Find().SetLimit(1000) + + res, err := mongoutil.Find[model.MsgDocModel](ctx, v.coll, bson.M{}, opt) + if err != nil { + panic(err) + } + ctx = context.Background() + for i := 0; i < 100000; i++ { + for j := range res { + res[j].DocID = strconv.FormatUint(rand.Uint64(), 10) + ":0" + } + if err := mongoutil.InsertMany(ctx, v.coll, res); err != nil { + panic(err) + } + t.Log("====>", time.Now(), i) + } + +} diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index 5167314da9..42507a6937 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -26,7 +26,7 @@ func Mongodb() *mongo.Database { func TestUserSeq(t *testing.T) { uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) - t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4)) + t.Log(uSeq.SetUserMinSeq(context.Background(), "1000", "2000", 4)) } func TestConversationSeq(t *testing.T) { @@ -35,3 +35,8 @@ func TestConversationSeq(t *testing.T) { t.Log(cSeq.Malloc(context.Background(), "2000", 10)) t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) } + +func TestUserGetUserReadSeqs(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"})) +} diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 7c9a8f1336..9faad416ae 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -88,6 +88,14 @@ func (s *seqUserMongo) GetUserReadSeq(ctx context.Context, conversationID string return s.getSeq(ctx, conversationID, userID, "read_seq") } +func (s *seqUserMongo) notFoundSet0(seq map[string]int64, conversationIDs []string) { + for _, conversationID := range conversationIDs { + if _, ok := seq[conversationID]; !ok { + seq[conversationID] = 0 + } + } +} + func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { if len(conversationID) == 0 { return map[string]int64{}, nil @@ -102,6 +110,7 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve for _, seq := range seqs { res[seq.ConversationID] = seq.ReadSeq } + s.notFoundSet0(res, conversationID) return res, nil } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b402f3ac7d..84f3a9e3e2 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -37,7 +37,7 @@ type Msg interface { GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error - SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) + SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) From a837fecda35b6f0237efdcb9f474aefcc0bfa9d6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 26 Jul 2024 16:23:45 +0800 Subject: [PATCH 33/91] merge: fix some typos in comments --- deployments/templates/env-template.yaml | 2 +- pkg/common/storage/controller/msg.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/templates/env-template.yaml b/deployments/templates/env-template.yaml index b1fb8b78be..b019f97e5b 100644 --- a/deployments/templates/env-template.yaml +++ b/deployments/templates/env-template.yaml @@ -129,7 +129,7 @@ REDIS_PORT=${REDIS_PORT} # Default: REDIS_PASSWORD=openIM123 REDIS_PASSWORD=${REDIS_PASSWORD} -# Kakfa username to authenticate with the Kafka service. +# Kafka username to authenticate with the Kafka service. # KAFKA_USERNAME=${KAFKA_USERNAME} # Port on which Kafka distributed streaming platform is running. diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 4ea74ef690..49268e0493 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -721,7 +721,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio } log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg)) if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() { - log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) + log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) } if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() { log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID) From 0a7992faab11e96b83a4ee4e3777dafca038ab71 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:30:58 +0800 Subject: [PATCH 34/91] Update env.template --- config/templates/env.template | 1 + 1 file changed, 1 insertion(+) diff --git a/config/templates/env.template b/config/templates/env.template index 5a232b2aef..a6bfcc820e 100644 --- a/config/templates/env.template +++ b/config/templates/env.template @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + # ----------------------------------------------------------------------------- # General Configuration # This section contains general configuration options for the entire environment. From 6c0c83eb3b5bdd96c135f6ec88f196b8fc1f2f55 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:32:36 +0800 Subject: [PATCH 35/91] Delete config/templates/env.template --- config/templates/env.template | 238 ---------------------------------- 1 file changed, 238 deletions(-) delete mode 100644 config/templates/env.template diff --git a/config/templates/env.template b/config/templates/env.template deleted file mode 100644 index a6bfcc820e..0000000000 --- a/config/templates/env.template +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright © 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# ----------------------------------------------------------------------------- -# General Configuration -# This section contains general configuration options for the entire environment. -# These options can be set via environment variables. If both environment variables -# and settings in this .env file exist, the environment variables take precedence. -# ----------------------------------------------------------------------------- -# ========================== -# General Configuration -# ========================== -# These settings apply to the overall environment. - -# Data storage directory for persistent data. -# Example: DATA_DIR=/path/to/data -DATA_DIR=/data/workspaces/open-im-server - -# Docker image registry. Uncomment the preferred one. -# Options: ghcr.io/openimsdk, openim, registry.cn-hangzhou.aliyuncs.com/openimsdk -# IMAGE_REGISTRY="ghcr.io/openimsdk" -# IMAGE_REGISTRY="openim" -# IMAGE_REGISTRY="registry.cn-hangzhou.aliyuncs.com/openimsdk" -IMAGE_REGISTRY=ghcr.io/openimsdk - -# ====================================== -# ========= Network Configuration ====== -# ====================================== - -# Subnet for the Docker network. -# Default: DOCKER_BRIDGE_SUBNET=172.28.0.0/16 -DOCKER_BRIDGE_SUBNET=172.28.0.0/16 - -# Set and specify the IP addresses of some containers. Generally speaking, -# you do not need to modify these configurations to facilitate debugging -DOCKER_BRIDGE_GATEWAY=172.28.0.1 -MONGO_NETWORK_ADDRESS=172.28.0.2 -REDIS_NETWORK_ADDRESS=172.28.0.3 -KAFKA_NETWORK_ADDRESS=172.28.0.4 -ZOOKEEPER_NETWORK_ADDRESS=172.28.0.5 -MINIO_NETWORK_ADDRESS=172.28.0.6 -OPENIM_WEB_NETWORK_ADDRESS=172.28.0.7 -OPENIM_SERVER_NETWORK_ADDRESS=172.28.0.8 -OPENIM_CHAT_NETWORK_ADDRESS=172.28.0.9 -PROMETHEUS_NETWORK_ADDRESS=172.28.0.10 -GRAFANA_NETWORK_ADDRESS=172.28.0.11 -NODE_EXPORTER_NETWORK_ADDRESS=172.28.0.12 -OPENIM_ADMIN_FRONT_NETWORK_ADDRESS=172.28.0.13 -ALERT_MANAGER_NETWORK_ADDRESS=172.28.0.14 - -# ============================================================================== -# Configuration Update Instructions -# ============================================================================== -# This header outlines the methods to update common variables in config.yaml and .env files. -# These instructions are vital for maintaining the OpenIM environment's configuration. -# -# METHOD 1: Regenerate All Configurations -# ---------------------------------------- -# Use this method to regenerate all configurations. -# Steps: -# 1. Delete existing config files: -# - openim-server/config/config.yaml -# - openim-chat/config/config.yaml -# 2. Modify the .env file as required. -# 3. Run 'docker compose up -d'. This will regenerate: -# - config/config.yaml -# -# METHOD 2: Modify Individual Configuration Files -# ----------------------------------------------- -# Use this method to update specific configuration files. -# Steps: -# 1. Modify the .env file as necessary. -# 2. Update the corresponding entries in: -# - config/config.yaml -# 3. Restart the services with 'docker compose up -d'. -# 4. Special Note: If you modify OPENIM_IP, API_OPENIM_PORT, or MINIO_PORT in .env, -# ensure to update the corresponding services and configurations accordingly. -# -# It is essential to follow these methods to ensure consistent and correct application behavior. -# ============================================================================== -# Local IP address of the service. Modify if necessary. -# Example: OPENIM_IP=172.28.0.1， -OPENIM_IP=127.0.0.1 - -# ----- ZooKeeper Configuration ----- -# Port for ZooKeeper service. -# Default: ZOOKEEPER_PORT=12181 -ZOOKEEPER_PORT=12181 - -# MongoDB service port configuration. -# Default: MONGO_PORT=37017 -MONGO_PORT=37017 - -# Password for MongoDB admin user. Used for service authentication. -# Default: MONGO_PASSWORD=openIM123 -MONGO_PASSWORD=openIM123 - -# Username for a regular OpenIM user in MongoDB. -# Default: MONGO_OPENIM_USERNAME=openIM -MONGO_OPENIM_USERNAME=openIM - -# Password for a regular OpenIM user in MongoDB. -# Default: MONGO_OPENIM_PASSWORD=openIM123456 -MONGO_OPENIM_PASSWORD=openIM123 - -# Specifies the database name to be used within MongoDB. -# Default: MONGO_DATABASE=openim_v3 -MONGO_DATABASE=openim_v3 - -MONGO_MAX_POOL_SIZE=100 -# ----- Redis Configuration ----- - -# Port on which Redis in-memory data structure store is running. -# Default: REDIS_PORT=16379 -REDIS_PORT=16379 - -# Password to authenticate with the Redis service. -# Default: REDIS_PASSWORD=openIM123 -REDIS_PASSWORD=openIM123 - -# Kafka username to authenticate with the Kafka service. -# KAFKA_USERNAME='' - -# Port on which Kafka distributed streaming platform is running. -# Default: KAFKA_PORT=19092 -KAFKA_PORT=19094 - -# Topic in Kafka for storing the latest messages in Redis. -# Default: KAFKA_LATESTMSG_REDIS_TOPIC=latestMsgToRedis -KAFKA_LATESTMSG_REDIS_TOPIC=latestMsgToRedis - -# MINIO_PORT -# ---------- -# MINIO_PORT sets the port for the MinIO object storage service. -# Upon changing this port, the MinIO endpoint URLs in the config/config.yaml file must be updated -# to reflect this change. The endpoints include both the 'endpoint' and 'signEndpoint' -# under the MinIO configuration. -# -# Default: MINIO_PORT=10005 -MINIO_PORT=10005 - -# Access key to authenticate with the MinIO service. -# Default: MINIO_ACCESS_KEY=root -# MINIO_ACCESS_KEY=root - -# Secret key corresponding to the access key for MinIO authentication. -# Default: MINIO_SECRET_KEY=openIM123 -MINIO_SECRET_KEY=openIM123 - -# ----- Prometheus Configuration ----- -# Port on which Prometheus service is running. -# Default: PROMETHEUS_PORT=19090 -PROMETHEUS_PORT=19090 - -# ----- Grafana Configuration ----- -# Port on which Grafana service is running. -# Default: GRAFANA_PORT=13000 -GRAFANA_PORT=13000 - -# ====================================== -# ============ OpenIM Web =============== -# ====================================== - -# Port on which OpenIM web service is running. -# Default: OPENIM_WEB_PORT=11001 -OPENIM_WEB_PORT=11001 - -# ====================================== -# ========= OpenIM Server ============== -# ====================================== -# Port for the OpenIM WebSockets. -# Default: OPENIM_WS_PORT=10001 -OPENIM_WS_PORT=10001 - -# API_OPENIM_PORT -# --------------- -# This variable defines the port on which the OpenIM API service will listen. -# When changing this port, it's essential to update the apiURL in the config.yaml file -# to ensure the API service is accessible at the new port. -# -# Default: API_OPENIM_PORT=10002 -API_OPENIM_PORT=10002 - -# ====================================== -# ========== OpenIM Chat =============== -# ====================================== - -# Branch name for OpenIM chat. -# Default: CHAT_IMAGE_VERSION=main -CHAT_IMAGE_VERSION=main - -# Port for the OpenIM chat API. -# Default: OPENIM_CHAT_API_PORT=10008 -OPENIM_CHAT_API_PORT=10008 - -# Port for the OpenIM admin API. -# Default: OPENIM_ADMIN_API_PORT=10009 -OPENIM_ADMIN_API_PORT=10009 - -# ====================================== -# ========== OpenIM Admin ============== -# ====================================== - -# Branch name for OpenIM server. -# Default: SERVER_IMAGE_VERSION=main -SERVER_IMAGE_VERSION=main - -# Port for the node exporter. -# Default: NODE_EXPORTER_PORT=19100 -NODE_EXPORTER_PORT=19100 - -# Port for the prometheus. -# Default: PROMETHEUS_PORT=19090 -PROMETHEUS_PORT=19090 - -# Port for the grafana. -# Default: GRAFANA_PORT=13000 -GRAFANA_PORT=13000 - -# Port for the admin front. -# Default: OPENIM_ADMIN_FRONT_PORT=11002 -OPENIM_ADMIN_FRONT_PORT=11002 - -# Port for the alertmanager. -# Default: ALERT_MANAGER_PORT=19093 -ALERT_MANAGER_PORT=19093 From e9b3a1952f7ab92846dab345ee97132c4e5b1dcd Mon Sep 17 00:00:00 2001 From: blooming <37789413+Bloomingg@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:36:11 +0800 Subject: [PATCH 36/91] cicd: add e2e in ci & before build docker image (#2346) * cicd: add e2e in ci & before build docker image * cicd: fix env minio addr * cicd: fix build image timing --- .github/workflows/build-docker-image.yml | 208 +++++++++++++---------- .github/workflows/link-pr.yml | 53 ------ .github/workflows/openimci.yml | 150 +++++----------- .github/workflows/stale.yml | 48 ------ 4 files changed, 169 insertions(+), 290 deletions(-) delete mode 100644 .github/workflows/link-pr.yml delete mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index b4733116e0..63982a012f 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -19,8 +19,27 @@ on: branches: - main - release-* + paths-ignore: + - "docs/**" + - "README.md" + - "README_zh-CN.md" + - "**.md" + - "docs/**" + - "CONTRIBUTING.md" tags: - v* + pull_request: + types: [closed] + branches: + - main + - release-* + paths-ignore: + - "docs/**" + - "README.md" + - "README_zh-CN.md" + - "**.md" + - "docs/**" + - "CONTRIBUTING.md" workflow_dispatch: env: @@ -29,109 +48,112 @@ env: jobs: build-dockerhub: - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true) runs-on: ubuntu-latest + if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.merged == false) }} steps: - - name: Checkout + - name: Checkout main repository uses: actions/checkout@v4 + with: + path: main-repo + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 -# docker.io/openim/openim-server:latest - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5.5.1 + - name: Build and push Docker image + uses: docker/build-push-action@v5 with: - images: openim/openim-server - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha + context: ./main-repo + load: true + tags: "openim/openim-server:local" - - name: Log in to Docker Hub - uses: docker/login-action@v3 + - name: Checkout compose repository + uses: actions/checkout@v4 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + repository: "openimsdk/openim-docker" + path: "compose-repo" - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - # linux/ppc64le,linux/s390x - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Get Internal IP Address + id: get-ip + run: | + IP=$(hostname -I | awk '{print $1}') + echo "The IP Address is: $IP" + echo "::set-output name=ip::$IP" - build-aliyun: - runs-on: ubuntu-latest - steps: - - name: Checkout + - name: Update .env to use the local image + run: | + sed -i 's|OPENIM_SERVER_IMAGE=.*|OPENIM_SERVER_IMAGE=openim/openim-server:local|' ${{ github.workspace }}/compose-repo/.env + sed -i 's|MINIO_EXTERNAL_ADDRESS=.*|MINIO_EXTERNAL_ADDRESS=http://${{ steps.get-ip.outputs.ip }}:10005|' ${{ github.workspace }}/compose-repo/.env + + - name: Start services using Docker Compose + run: | + cd ${{ github.workspace }}/compose-repo + docker compose up -d + sleep 30 + + - name: Check openim-server health + run: | + timeout=300 + interval=30 + elapsed=0 + while [[ $elapsed -le $timeout ]]; do + if ! docker exec openim-server mage check; then + echo "openim-server is not ready, waiting..." + sleep $interval + elapsed=$(($elapsed + $interval)) + else + echo "Health check successful" + exit 0 + fi + done + echo "Health check failed after 5 minutes" + exit 1 + + - name: Check openim-chat health + if: success() + run: | + if ! docker exec openim-chat mage check; then + echo "openim-chat check failed" + exit 1 + else + echo "Health check successful" + exit 0 + fi + + - name: Checkout e2e + if: success() uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 -# registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server:latest - - name: Extract metadata (tags, labels) for Docker - id: meta2 - uses: docker/metadata-action@v5.5.1 with: - images: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha + repository: "openimsdk/test-e2e" + path: e2e-repo - - name: Log in to AliYun Docker Hub - uses: docker/login-action@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 with: - registry: registry.cn-hangzhou.aliyuncs.com - username: ${{ secrets.ALIREGISTRY_USERNAME }} - password: ${{ secrets.ALIREGISTRY_TOKEN }} + python-version: '3.9' - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - # linux/ppc64le,linux/s390x - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta2.outputs.tags }} - labels: ${{ steps.meta2.outputs.labels }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y xvfb libxi6 libgconf-2-4 + cd ${{ github.workspace }}/e2e-repo + pip install -r requirements.txt + + - name: Run tests + run: | + cd ${{ github.workspace }}/e2e-repo + xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script - build-ghcr: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 -# ghcr.io/openimsdk/openim-server:latest - name: Extract metadata (tags, labels) for Docker - id: meta3 + if: success() + id: meta uses: docker/metadata-action@v5.5.1 with: - images: ghcr.io/openimsdk/openim-server + images: | + openim/openim-server + ghcr.io/openimsdk/openim-server + registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server # generate Docker tags based on the following events/attributes tags: | type=ref,event=tag @@ -144,19 +166,33 @@ jobs: type=semver,pattern={{major}} type=sha + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Log in to Aliyun Container Registry + uses: docker/login-action@v2 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.ALIREGISTRY_USERNAME }} + password: ${{ secrets.ALIREGISTRY_TOKEN }} + - name: Build and push Docker image uses: docker/build-push-action@v5 with: - context: . + context: ./main-repo + push: true # linux/ppc64le,linux/s390x platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta3.outputs.tags }} - labels: ${{ steps.meta3.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + diff --git a/.github/workflows/link-pr.yml b/.github/workflows/link-pr.yml deleted file mode 100644 index 6c5ff6c5b9..0000000000 --- a/.github/workflows/link-pr.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Rebot for Link check error - -# Every Monday at 12:30 p.m -on: - schedule: - - cron: '30 12 * * 1' - -jobs: - linkChecker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Link Checker - id: lychee - uses: lycheeverse/lychee-action@v1.9.3 - with: - # For parameter description, see https://github.com/lycheeverse/lychee#commandline-parameters - # Actions Link address -> https://github.com/lycheeverse/lychee-action - # -E, --exclude-all-private Exclude all private IPs from checking. - # -i, --insecure Proceed for server connections considered insecure (invalid TLS) - # -n, --no-progress Do not show progress bar. - # -t, --timeout Website timeout in seconds from connect to response finished [default:20] - # --max-concurrency Maximum number of concurrent network requests [default: 128] - # -a --accept Comma-separated list of accepted status codes for valid links - # docs/.vitepress/dist the site directory to check - # ./*.md all markdown files in the root directory - args: --verbose -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md' - env: - GITHUB_TOKEN: ${{secrets.BOT_GITHUB_TOKEN}} - - - name: Create Issue From File - if: env.lychee_exit_code != 0 - uses: peter-evans/create-issue-from-file@v5 - with: - title: Bug reports for links in OpenIM docs - content-filepath: ./lychee/out.md - labels: kind/documentation, triage/unresolved, report - token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index 83d495a0e4..9952a13812 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -71,6 +71,18 @@ jobs: run: sudo bash bootstrap.sh timeout-minutes: 20 + - name: Get Internal IP Address + id: get-ip + run: | + IP=$(hostname -I | awk '{print $1}') + echo "The IP Address is: $IP" + echo "::set-output name=ip::$IP" + + - name: Update .env + run: | + sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + cat config/minio.yml + - name: Build, Start, Check Services and Print Logs for Linux run: | sudo mage @@ -84,108 +96,40 @@ jobs: sudo mage start sudo mage check + - name: Checkout chat repository + uses: actions/checkout@v4 + with: + repository: 'openimsdk/chat' + path: 'chat-repo' -# build-mac: -# name: Execute OpenIM Script On macOS -# runs-on: macos-latest -# permissions: -# contents: write -# pull-requests: write -# environment: -# name: openim -# strategy: -# matrix: -# arch: [arm64, armv7, amd64] -# -# steps: -# - uses: actions/checkout@v3 - -# - name: Set up Go -# uses: actions/setup-go@v4 -# with: -# go-version: '1.21' - - -# while ! docker system info > /dev/null 2>&1; do -# echo "Waiting for Docker to start..." -# sleep 10 # Increased delay to ensure Docker starts properly -# done - -# - name: Install Docker -# run: | -# brew install docker -# brew install docker-compose -# sleep 10 -# docker-compose up -d -# sleep 30 -# timeout-minutes: 20 -# - -# - name: init -# run: sudo bash bootstrap.sh -# timeout-minutes: 20 - -# - name: Build, Start, Check Services and Print Logs for Linux -# run: | -# sudo mage -# sudo mage start -# sudo mage check - -# - name: Restart Services and Print Logs -# run: | -# sudo mage stop -# sudo mage start -# sudo mage check - -# build-windows: -# name: Execute OpenIM Script On Windows -# runs-on: windows-latest -# permissions: -# contents: write -# pull-requests: write -# environment: -# name: openim -# strategy: -# matrix: -# arch: [arm64, armv7, amd64] -# -# steps: -# - uses: actions/checkout@v3 - -# - name: Set up Go -# uses: actions/setup-go@v4 -# with: -# go-version: '1.21' - -# - name: Set up Docker for Windows -# run: | -# $images = @("zookeeper", "redis", "kafka") -# foreach ($image in $images) { -# $tag = "$image:latest" -# docker pull $tag | Out-Null -# if ($LASTEXITCODE -ne 0) { -# Write-Host "Skipping $image as it is not available for Windows" -# } else { -# Write-Host "Successfully pulled $image" -# } -# } -# docker compose up -d -# Start-Sleep -Seconds 30 -# timeout-minutes: 20 -# shell: pwsh - -# - name: init -# run: bootstrap.bat -# timeout-minutes: 20 - -# - name: Build, Start, Check Services and Print Logs for Linux -# run: | -# mage -# mage start -# mage check + - name: Build and Start Chat Services + run: | + cd ${{ github.workspace }}/chat-repo + sudo mage + sudo mage start + sudo mage check + + - name: Checkout e2e repository + uses: actions/checkout@v4 + with: + repository: "openimsdk/test-e2e" + path: e2e-repo -# - name: Restart Services and Print Logs -# run: | -# mage stop -# mage start -# mage check + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y xvfb libxi6 libgconf-2-4 + cd ${{ github.workspace }}/e2e-repo + pip install -r requirements.txt + + - name: Run tests + run: | + cd ${{ github.workspace }}/e2e-repo + xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script + + \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 4445f6ddaa..0000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. -# -# You can adjust the behavior by modifying this file. -# For more information, see: -# https://github.com/actions/stale -name: Mark stale issues and pull requests - -on: - schedule: - - cron: '0 8 * * 1' - -jobs: - stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v9 - with: - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} - days-before-stale: 60 - days-before-close: 305 - stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days.' - stale-pr-message: 'This issue is stale because it has been open 60 days with no activity.' - close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.' - close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity. You can reopen it if you want.' - stale-pr-label: lifecycle/stale - stale-issue-label: lifecycle/stale - exempt-issue-labels: 'openim' - exempt-pr-labels: 'openim' - exempt-draft-pr: true From ff66e972219a50e9065da24a5756269fcb0b861c Mon Sep 17 00:00:00 2001 From: blooming <37789413+Bloomingg@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:20:54 +0800 Subject: [PATCH 37/91] fix: rm e2e in ci (#2449) --- .github/workflows/build-docker-image.yml | 42 ++++++++--------- .github/workflows/openimci.yml | 58 ++++++++++++------------ 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 63982a012f..d0b9dddbc4 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -121,29 +121,29 @@ jobs: exit 0 fi - - name: Checkout e2e - if: success() - uses: actions/checkout@v4 - with: - repository: "openimsdk/test-e2e" - path: e2e-repo - - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: '3.9' + # - name: Checkout e2e + # if: success() + # uses: actions/checkout@v4 + # with: + # repository: "openimsdk/test-e2e" + # path: e2e-repo + + # - name: Set up Python 3.9 + # uses: actions/setup-python@v4 + # with: + # python-version: '3.9' - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y xvfb libxi6 libgconf-2-4 - cd ${{ github.workspace }}/e2e-repo - pip install -r requirements.txt + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y xvfb libxi6 libgconf-2-4 + # cd ${{ github.workspace }}/e2e-repo + # pip install -r requirements.txt - - name: Run tests - run: | - cd ${{ github.workspace }}/e2e-repo - xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script + # - name: Run tests + # run: | + # cd ${{ github.workspace }}/e2e-repo + # xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script - name: Extract metadata (tags, labels) for Docker if: success() diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index 9952a13812..8f3630dd08 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -71,17 +71,17 @@ jobs: run: sudo bash bootstrap.sh timeout-minutes: 20 - - name: Get Internal IP Address - id: get-ip - run: | - IP=$(hostname -I | awk '{print $1}') - echo "The IP Address is: $IP" - echo "::set-output name=ip::$IP" + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" - - name: Update .env - run: | - sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml - cat config/minio.yml + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml - name: Build, Start, Check Services and Print Logs for Linux run: | @@ -109,27 +109,27 @@ jobs: sudo mage start sudo mage check - - name: Checkout e2e repository - uses: actions/checkout@v4 - with: - repository: "openimsdk/test-e2e" - path: e2e-repo + # - name: Checkout e2e repository + # uses: actions/checkout@v4 + # with: + # repository: "openimsdk/test-e2e" + # path: e2e-repo - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: '3.9' + # - name: Set up Python 3.9 + # uses: actions/setup-python@v4 + # with: + # python-version: '3.9' - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y xvfb libxi6 libgconf-2-4 - cd ${{ github.workspace }}/e2e-repo - pip install -r requirements.txt + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y xvfb libxi6 libgconf-2-4 + # cd ${{ github.workspace }}/e2e-repo + # pip install -r requirements.txt - - name: Run tests - run: | - cd ${{ github.workspace }}/e2e-repo - xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script + # - name: Run tests + # run: | + # cd ${{ github.workspace }}/e2e-repo + # xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script \ No newline at end of file From 220a01d7f8d99fc2c1fb2cddca69ba44e96c2150 Mon Sep 17 00:00:00 2001 From: printlin <32053356+printlin@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:24:25 +0800 Subject: [PATCH 38/91] fix: fill notification offlinePush by config (#2422) * fix: fill notification offlinePush by config * fix: fill notification OfflinePush by config --- pkg/rpcclient/msg.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index f660c74dd7..124cc49af3 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -324,6 +324,10 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co options := config.GetOptionsByNotification(optionsConfig) s.SetOptionsByContentType(ctx, options, contentType) msg.Options = options + // fill Notification OfflinePush by config + offlineInfo.Title = optionsConfig.OfflinePush.Title + offlineInfo.Desc = optionsConfig.OfflinePush.Desc + offlineInfo.Ex = optionsConfig.OfflinePush.Ext msg.OfflinePushInfo = &offlineInfo req.MsgData = &msg _, err = s.sendMsg(ctx, &req) From ef46abd19364e2b1abf5226bac078cbb7a3a90fa Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sun, 28 Jul 2024 11:18:23 +0800 Subject: [PATCH 39/91] chore: add Warn log in writePongMsg. (#2452) * update protocol in go mod. * add debug log in writePongMsg. * update log level. * add Warn log in writePongMsg. * add debug log. --- internal/msggateway/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index ded830c43a..9fd71d9893 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -393,6 +393,7 @@ func (c *Client) writePingMsg() error { func (c *Client) writePongMsg(appData string) error { log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) if c.closed.Load() { + log.ZWarn(c.ctx, "is closed in server", nil, "appdata", appData, "closed err", c.closedErr) return nil } @@ -403,6 +404,7 @@ func (c *Client) writePongMsg(appData string) error { log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) err := c.conn.SetWriteDeadline(writeWait) if err != nil { + log.ZWarn(c.ctx, "SetWriteDeadline in Server have error", errs.Wrap(err), "writeWait", writeWait, "appData", appData) return errs.Wrap(err) } err = c.conn.WriteMessage(PongMessage, []byte(appData)) @@ -410,5 +412,6 @@ func (c *Client) writePongMsg(appData string) error { log.ZWarn(c.ctx, "Write Message have error", errs.Wrap(err), "Pong msg", PongMessage) } + log.ZDebug(c.ctx, "write message is success", "appdata", appData, "closed err", c.closedErr) return errs.Wrap(err) } From b0cc4373a53ce8ac7bc12b15067f2338b80c83ba Mon Sep 17 00:00:00 2001 From: printlin <32053356+printlin@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:08:28 +0800 Subject: [PATCH 40/91] fix: #2410 BeforeMemberJoinGroup callback member error (#2423) --- internal/rpc/group/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index e3d1d4dfe6..6790d09581 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -442,7 +442,7 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, member, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } groupMembers = append(groupMembers, member) From 231aac2b8a6ed9c4e9c175b23b4d4eaa4aae93ae Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:37:22 +0800 Subject: [PATCH 41/91] fix: search log can return platform (#2456) * fix: search log can return platform * fix: change platform type to string --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/third/log.go | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 71301d2909..1a1cf36d2d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.42 + github.com/openimsdk/protocol v0.0.69-alpha.47 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 53060b1984..815afe8d21 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.42 h1:Vwuru2NtyTHuqaM+1JGxcoGvP25QWjS92oI0zGJp+lM= -github.com/openimsdk/protocol v0.0.69-alpha.42/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.47 h1:WEpU7dHSzcpiyPoUkgSt1mC9HfQ6xSDNNZf4KWbZiFI= +github.com/openimsdk/protocol v0.0.69-alpha.47/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index cd52727cba..68d7088b0c 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -26,7 +26,6 @@ import ( "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/stringutil" ) func genLogID() string { @@ -111,7 +110,7 @@ func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo { return &third.LogInfo{ Filename: log.FileName, UserID: log.UserID, - Platform: stringutil.StringToInt32(log.Platform), + Platform: log.Platform, Url: log.Url, CreateTime: log.CreateTime.UnixMilli(), LogID: log.LogID, From ed0ab58a9e6f5ff29b7956894bf15deb30ead8ff Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:53:26 +0800 Subject: [PATCH 42/91] fix ImportFriends (#2458) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends --------- Co-authored-by: withchao --- pkg/common/storage/controller/friend.go | 61 +++++++++++++++---------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index e402f5980b..636371198a 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -152,42 +152,55 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return f.tx.Transaction(ctx, func(ctx context.Context) error { cache := f.cache.CloneFriendCache() // user find friends - fs1, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) + myFriends, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - opUserID := mcontext.GetOperationID(ctx) - for _, v := range friendUserIDs { - fs1 = append(fs1, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: v, AddSource: addSource, OperatorUserID: opUserID}) - } - fs11 := datautil.DistinctAny(fs1, func(e *model.Friend) string { - return e.FriendUserID - }) - - err = f.friend.Create(ctx, fs11) + addOwners, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - fs2, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) - if err != nil { - return err + opUserID := mcontext.GetOperationID(ctx) + friends := make([]*model.Friend, 0, len(friendUserIDs)*2) + myFriendsSet := datautil.SliceSetAny(myFriends, func(friend *model.Friend) string { + return friend.FriendUserID + }) + addOwnersSet := datautil.SliceSetAny(addOwners, func(friend *model.Friend) string { + return friend.OwnerUserID + }) + newMyFriendIDs := make([]string, 0, len(friendUserIDs)) + newMyOwnerIDs := make([]string, 0, len(friendUserIDs)) + for _, userID := range friendUserIDs { + if ownerUserID == userID { + continue + } + if _, ok := myFriendsSet[userID]; !ok { + myFriendsSet[userID] = struct{}{} + newMyFriendIDs = append(newMyFriendIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: userID, AddSource: addSource, OperatorUserID: opUserID}) + } + if _, ok := addOwnersSet[userID]; !ok { + addOwnersSet[userID] = struct{}{} + newMyOwnerIDs = append(newMyOwnerIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: userID, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) + } } - var newFriendIDs []string - for _, v := range friendUserIDs { - fs2 = append(fs2, &model.Friend{OwnerUserID: v, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) - newFriendIDs = append(newFriendIDs, v) + if len(friends) == 0 { + return nil } - fs22 := datautil.DistinctAny(fs2, func(e *model.Friend) string { - return e.OwnerUserID - }) - err = f.friend.Create(ctx, fs22) + err = f.friend.Create(ctx, friends) if err != nil { return err } - newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...) + if len(newMyFriendIDs) > 0 { + cache = cache.DelFriendIDs(newMyFriendIDs...) + cache = cache.DelFriends(ownerUserID, newMyFriendIDs).DelMaxFriendVersion(newMyFriendIDs...) + } + if len(newMyOwnerIDs) > 0 { + cache = cache.DelFriendIDs(newMyOwnerIDs...) + cache = cache.DelOwner(ownerUserID, newMyOwnerIDs).DelMaxFriendVersion(newMyOwnerIDs...) + } return cache.ChainExecDel(ctx) - }) } From fb689618d88a19e3529f386d3026ecde0ec99185 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 30 Jul 2024 18:09:52 +0800 Subject: [PATCH 43/91] feat: update webhookBeforeMemberJoinGroup to batch method. (#2459) * update protocol in go mod. * add debug log in writePongMsg. * update log level. * add Warn log in writePongMsg. * add debug log. * feat: update webhookBeforeMemberJoinGroup to batch method. * feat: update version field implement. * update webhook implement contents. * update method field and contents. * update callbackCommand field. * fix: add correct fields. * update struct tags. --- .env | 1 - internal/msggateway/client.go | 3 -- internal/rpc/group/callback.go | 58 +++++++++++++++++++++++----------- internal/rpc/group/group.go | 40 +++++++++++------------ pkg/callbackstruct/constant.go | 2 +- pkg/callbackstruct/group.go | 23 ++++++++++---- pkg/common/cmd/api.go | 5 +-- pkg/common/cmd/auth.go | 5 +-- pkg/common/cmd/conversation.go | 5 +-- pkg/common/cmd/cron_task.go | 5 +-- pkg/common/cmd/friend.go | 5 +-- pkg/common/cmd/group.go | 5 +-- pkg/common/cmd/msg.go | 5 +-- pkg/common/cmd/msg_gateway.go | 4 +-- pkg/common/cmd/msg_transfer.go | 5 +-- pkg/common/cmd/push.go | 5 +-- pkg/common/cmd/root.go | 5 +-- pkg/common/cmd/third.go | 5 +-- pkg/common/cmd/user.go | 5 +-- pkg/common/config/parse.go | 4 --- pkg/common/config/version | 1 - version/version | 1 + version/version.go | 6 ++++ 23 files changed, 122 insertions(+), 81 deletions(-) delete mode 100644 pkg/common/config/version create mode 100644 version/version create mode 100644 version/version.go diff --git a/.env b/.env index 3199b37149..a71332d5ee 100644 --- a/.env +++ b/.env @@ -1,4 +1,3 @@ - MONGO_IMAGE=mongo:6.0.2 REDIS_IMAGE=redis:7.0.0 ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 9fd71d9893..a4902570a6 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -397,11 +397,9 @@ func (c *Client) writePongMsg(appData string) error { return nil } - log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) c.w.Lock() defer c.w.Unlock() - log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) err := c.conn.SetWriteDeadline(writeWait) if err != nil { log.ZWarn(c.ctx, "SetWriteDeadline in Server have error", errs.Wrap(err), "writeWait", writeWait, "appData", appData) @@ -412,6 +410,5 @@ func (c *Client) writePongMsg(appData string) error { log.ZWarn(c.ctx, "Write Message have error", errs.Wrap(err), "Pong msg", PongMessage) } - log.ZDebug(c.ctx, "write message is success", "appdata", appData, "closed err", c.closedErr) return errs.Wrap(err) } diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index f31c4587c5..f877aa64a8 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -16,6 +16,8 @@ package group import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" - "time" ) // CallbackBeforeCreateGroup callback before create group. @@ -100,27 +101,45 @@ func (s *groupServer) webhookAfterCreateGroup(ctx context.Context, after *config s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateGroupResp{}, after) } -func (s *groupServer) webhookBeforeMemberJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMember *model.GroupMember, groupEx string) error { +func (s *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMembers []*model.GroupMember, groupID string, groupEx string) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - cbReq := &callbackstruct.CallbackBeforeMemberJoinGroupReq{ - CallbackCommand: callbackstruct.CallbackBeforeMemberJoinGroupCommand, - GroupID: groupMember.GroupID, - UserID: groupMember.UserID, - Ex: groupMember.Ex, + groupMembersMap := datautil.SliceToMap(groupMembers, func(e *model.GroupMember) string { + return e.UserID + }) + var groupMembersCallback []*callbackstruct.CallbackGroupMember + + for _, member := range groupMembers { + groupMembersCallback = append(groupMembersCallback, &callbackstruct.CallbackGroupMember{ + UserID: member.UserID, + Ex: member.Ex, + }) + } + + cbReq := &callbackstruct.CallbackBeforeMembersJoinGroupReq{ + CallbackCommand: callbackstruct.CallbackBeforeMembersJoinGroupCommand, + GroupID: groupID, + MembersList: groupMembersCallback, GroupEx: groupEx, } - resp := &callbackstruct.CallbackBeforeMemberJoinGroupResp{} + resp := &callbackstruct.CallbackBeforeMembersJoinGroupResp{} + if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } - if resp.MuteEndTime != nil { - groupMember.MuteEndTime = time.UnixMilli(*resp.MuteEndTime) + for _, memberCallbackResp := range resp.MemberCallbackList { + if _, ok := groupMembersMap[(*memberCallbackResp.UserID)]; ok { + if memberCallbackResp.MuteEndTime != nil { + groupMembersMap[(*memberCallbackResp.UserID)].MuteEndTime = time.UnixMilli(*memberCallbackResp.MuteEndTime) + } + + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].FaceURL, memberCallbackResp.FaceURL) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].Ex, memberCallbackResp.Ex) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].Nickname, memberCallbackResp.Nickname) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].RoleLevel, memberCallbackResp.RoleLevel) + } } - datautil.NotNilReplace(&groupMember.FaceURL, resp.FaceURL) - datautil.NotNilReplace(&groupMember.Ex, resp.Ex) - datautil.NotNilReplace(&groupMember.Nickname, resp.Nickname) - datautil.NotNilReplace(&groupMember.RoleLevel, resp.RoleLevel) + return nil }) } @@ -244,10 +263,13 @@ func (s *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before return err } - if len(resp.RefusedMembersAccount) > 0 { - // Handle the scenario where certain members are refused - // You might want to update the req.Members list or handle it as per your business logic - } + // Handle the scenario where certain members are refused + // You might want to update the req.Members list or handle it as per your business logic + + // if len(resp.RefusedMembersAccount) > 0 { + // implement members are refused + // } + return nil }) } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 6790d09581..aa12c9d0f1 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -246,7 +246,7 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, err } - joinGroup := func(userID string, roleLevel int32) error { + joinGroupFunc := func(userID string, roleLevel int32) { groupMember := &model.GroupMember{ GroupID: group.GroupID, UserID: userID, @@ -258,25 +258,23 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { - return err - } groupMembers = append(groupMembers, groupMember) - return nil - } - if err := joinGroup(req.OwnerUserID, constant.GroupOwner); err != nil { - return nil, err } + + joinGroupFunc(req.OwnerUserID, constant.GroupOwner) + for _, userID := range req.AdminUserIDs { - if err := joinGroup(userID, constant.GroupAdmin); err != nil { - return nil, err - } + joinGroupFunc(userID, constant.GroupAdmin) } + for _, userID := range req.MemberUserIDs { - if err := joinGroup(userID, constant.GroupOrdinaryUsers); err != nil { - return nil, err - } + joinGroupFunc(userID, constant.GroupOrdinaryUsers) } + + if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + if err := s.db.CreateGroup(ctx, []*model.Group{group}, groupMembers); err != nil { return nil, err } @@ -442,12 +440,13 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, member, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { - return nil, err - } groupMembers = append(groupMembers, member) + } + if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err } + if err := s.db.CreateGroup(ctx, nil, groupMembers); err != nil { return nil, err } @@ -811,9 +810,9 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup MuteEndTime: time.Unix(0, 0), InviterUserID: groupRequest.InviterUserID, OperatorUserID: mcontext.GetOpUserID(ctx), - Ex: groupRequest.Ex, } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, member, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + + if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{member}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } } @@ -882,7 +881,7 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{groupMember}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } @@ -898,6 +897,7 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) return &pbgroup.JoinGroupResp{}, nil } + groupRequest := model.GroupRequest{ UserID: req.InviterUserID, ReqMsg: req.ReqMessage, diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index 66e1598cda..ab393dd36e 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -56,7 +56,7 @@ const ( CallbackBeforeUpdateUserInfoCommand = "callbackBeforeUpdateUserInfoCommand" CallbackBeforeCreateGroupCommand = "callbackBeforeCreateGroupCommand" CallbackAfterCreateGroupCommand = "callbackAfterCreateGroupCommand" - CallbackBeforeMemberJoinGroupCommand = "callbackBeforeMemberJoinGroupCommand" + CallbackBeforeMembersJoinGroupCommand = "callbackBeforeMembersJoinGroupCommand" CallbackBeforeSetGroupMemberInfoCommand = "callbackBeforeSetGroupMemberInfoCommand" CallbackAfterSetGroupMemberInfoCommand = "callbackAfterSetGroupMemberInfoCommand" ) diff --git a/pkg/callbackstruct/group.go b/pkg/callbackstruct/group.go index e78d45ab4c..23a73ebd23 100644 --- a/pkg/callbackstruct/group.go +++ b/pkg/callbackstruct/group.go @@ -59,16 +59,20 @@ type CallbackAfterCreateGroupResp struct { CommonCallbackResp } -type CallbackBeforeMemberJoinGroupReq struct { +type CallbackGroupMember struct { + UserID string `json:"userID"` + Ex string `json:"ex"` +} + +type CallbackBeforeMembersJoinGroupReq struct { CallbackCommand `json:"callbackCommand"` - GroupID string `json:"groupID"` - UserID string `json:"userID"` - Ex string `json:"ex"` - GroupEx string `json:"groupEx"` + GroupID string `json:"groupID"` + MembersList []*CallbackGroupMember `json:"memberList"` + GroupEx string `json:"groupEx"` } -type CallbackBeforeMemberJoinGroupResp struct { - CommonCallbackResp +type MemberJoinGroupCallBack struct { + UserID *string `json:"userID"` Nickname *string `json:"nickname"` FaceURL *string `json:"faceURL"` RoleLevel *int32 `json:"roleLevel"` @@ -76,6 +80,11 @@ type CallbackBeforeMemberJoinGroupResp struct { Ex *string `json:"ex"` } +type CallbackBeforeMembersJoinGroupResp struct { + CommonCallbackResp + MemberCallbackList []*MemberJoinGroupCallBack `json:"memberCallbackList"` +} + type CallbackBeforeSetGroupMemberInfoReq struct { CallbackCommand `json:"callbackCommand"` GroupID string `json:"groupID"` diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index ecdb0dd3ad..4088ecd09d 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/api" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -38,7 +39,7 @@ func NewApiCmd() *ApiCmd { DiscoveryConfigFilename: &apiConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 7d75a7da65..b35a95f395 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/auth" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -40,7 +41,7 @@ func NewAuthRpcCmd() *AuthRpcCmd { DiscoveryConfigFilename: &authConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 57ffa52bc4..bdb4447f48 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/conversation" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -43,7 +44,7 @@ func NewConversationRpcCmd() *ConversationRpcCmd { DiscoveryConfigFilename: &conversationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index fd44475246..d6c5e472e1 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/tools" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -38,7 +39,7 @@ func NewCronTaskCmd() *CronTaskCmd { DiscoveryConfigFilename: &cronTaskConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 8be1f77458..1bc9e6c543 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -44,7 +45,7 @@ func NewFriendRpcCmd() *FriendRpcCmd { DiscoveryConfigFilename: &friendConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 20124be957..9b0fbf8de3 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -16,10 +16,11 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/group" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewGroupRpcCmd() *GroupRpcCmd { DiscoveryConfigFilename: &groupConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index 91f7931fbe..bfd29398ef 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/msg" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewMsgRpcCmd() *MsgRpcCmd { DiscoveryConfigFilename: &msgConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 29d3fba330..6363bfbf9e 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -16,9 +16,9 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/internal/msggateway" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -42,7 +42,7 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 0d48281e50..3643934135 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -42,7 +43,7 @@ func NewMsgTransferCmd() *MsgTransferCmd { DiscoveryConfigFilename: &msgTransferConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 6e6014021e..c9b8b1c245 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/push" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewPushRpcCmd() *PushRpcCmd { DiscoveryConfigFilename: &pushConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { ret.pushConfig.FcmConfigPath = ret.ConfigPath() return ret.runE() diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 84e9856973..b43f86557f 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -19,6 +19,7 @@ import ( "path/filepath" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/spf13/cobra" @@ -138,13 +139,13 @@ func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { r.log.StorageLocation, r.log.RemainRotationCount, r.log.RotationTime, - config.Version, + version.Version, r.log.IsSimplify, ) if err != nil { return errs.Wrap(err) } - return errs.Wrap(log.InitConsoleLogger(r.processName, r.log.RemainLogLevel, r.log.IsJson, config.Version)) + return errs.Wrap(log.InitConsoleLogger(r.processName, r.log.RemainLogLevel, r.log.IsJson, version.Version)) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index b6731f1ff4..a301b738fa 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/third" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -44,7 +45,7 @@ func NewThirdRpcCmd() *ThirdRpcCmd { DiscoveryConfigFilename: &thirdConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 674f9e3a6d..9a614afcab 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/user" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewUserRpcCmd() *UserRpcCmd { DiscoveryConfigFilename: &userConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index 28e9f5db60..08f82ac7d3 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -15,7 +15,6 @@ package config import ( - _ "embed" "os" "path/filepath" @@ -26,9 +25,6 @@ import ( "gopkg.in/yaml.v3" ) -//go:embed version -var Version string - const ( FileName = "config.yaml" NotificationFileName = "notification.yaml" diff --git a/pkg/common/config/version b/pkg/common/config/version deleted file mode 100644 index 240bba9069..0000000000 --- a/pkg/common/config/version +++ /dev/null @@ -1 +0,0 @@ -3.7.0 \ No newline at end of file diff --git a/version/version b/version/version new file mode 100644 index 0000000000..0be1fc7d24 --- /dev/null +++ b/version/version @@ -0,0 +1 @@ +3.8.0 \ No newline at end of file diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000000..23b3a82f53 --- /dev/null +++ b/version/version.go @@ -0,0 +1,6 @@ +package version + +import _ "embed" + +//go:embed version +var Version string From d2b02dbabecc9934271f4a7cc5b7f3e5cbfd5cd9 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:50:55 +0800 Subject: [PATCH 44/91] feat: add some logs (#2461) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- internal/push/push_handler.go | 1 + pkg/common/storage/cache/redis/online.go | 6 +- pkg/rpccache/online.go | 76 +++++++++++++----------- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ed87b3929f..c3b27372b3 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -198,6 +198,7 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. offlineUserIDs = append(offlineUserIDs, userID) } } + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { var err error diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index dc6a5f775b..a012e1cd2d 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -5,6 +5,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "strconv" "time" @@ -82,8 +83,11 @@ func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, o argv = append(argv, platformID) } keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} - if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + status, err := s.rdb.Eval(ctx, script, keys, argv).Result() + if err != nil { + log.ZError(ctx, "redis SetUserOnline", err, "userID", userID, "online", online, "offline", offline) return err } + log.ZDebug(ctx, "redis SetUserOnline", "userID", userID, "online", online, "offline", offline, "status", status) return nil } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5db68d198d..372a7efe82 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -28,7 +28,7 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) if err != nil { - log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + log.ZError(ctx, "OnlineCache setUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } storageCache := x.setUserOnline(userID, platformIDs) @@ -48,9 +48,15 @@ type OnlineCache struct { } func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - return o.local.Get(userID, func() ([]int32, error) { + platformIDs, err := o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) + return nil, err + } + log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + return nil, err } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { @@ -61,39 +67,39 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } -func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { - onlineUserIDs := make([]string, 0, len(userIDs)) - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) - return onlineUserIDs, nil -} - -func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { - userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - var onlineUserIDs []string - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) - return onlineUserIDs, nil -} +//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { +// onlineUserIDs := make([]string, 0, len(userIDs)) +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) +// return onlineUserIDs, nil +//} +// +//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// var onlineUserIDs []string +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) +// return onlineUserIDs, nil +//} func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { return o.local.SetHas(userID, platformIDs) From 9cfbc3aaffa0ce5b81c2f8701b8205f30be74eb5 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:21:19 +0800 Subject: [PATCH 45/91] fix: return value (#2462) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs * add some logs --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- pkg/rpccache/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 372a7efe82..13578a7df6 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -56,7 +56,7 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return nil, err } log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) - return nil, err + return platformIDs, nil } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { From 5dac91569de904efd619f165ef87089c428ed1c1 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 31 Jul 2024 18:12:19 +0800 Subject: [PATCH 46/91] refactor: rename friend module to relation. (#2463) * update protocol in go mod. * add debug log in writePongMsg. * update log level. * add Warn log in writePongMsg. * add debug log. * feat: update webhookBeforeMemberJoinGroup to batch method. * feat: update version field implement. * update webhook implement contents. * update method field and contents. * update callbackCommand field. * fix: add correct fields. * update struct tags. * refactor: rename friend module to relation. --- internal/rpc/{friend => relation}/black.go | 2 +- internal/rpc/{friend => relation}/callback.go | 2 +- internal/rpc/{friend => relation}/friend.go | 3 +- .../rpc/{friend => relation}/notification.go | 2 +- internal/rpc/{friend => relation}/sync.go | 2 +- internal/rpc/user/user.go | 6 ++-- pkg/common/cmd/friend.go | 34 +++++++++---------- 7 files changed, 26 insertions(+), 25 deletions(-) rename internal/rpc/{friend => relation}/black.go (99%) rename internal/rpc/{friend => relation}/callback.go (99%) rename internal/rpc/{friend => relation}/friend.go (99%) rename internal/rpc/{friend => relation}/notification.go (99%) rename internal/rpc/{friend => relation}/sync.go (99%) diff --git a/internal/rpc/friend/black.go b/internal/rpc/relation/black.go similarity index 99% rename from internal/rpc/friend/black.go rename to internal/rpc/relation/black.go index 218d1e7f85..e149e31653 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/relation/black.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/callback.go b/internal/rpc/relation/callback.go similarity index 99% rename from internal/rpc/friend/callback.go rename to internal/rpc/relation/callback.go index 746ad21fa1..69c4c9e0ec 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/relation/callback.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/friend.go b/internal/rpc/relation/friend.go similarity index 99% rename from internal/rpc/friend/friend.go rename to internal/rpc/relation/friend.go index bdb786bca4..3d29ad3379 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/relation/friend.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" + "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/open-im-server/v3/pkg/common/config" diff --git a/internal/rpc/friend/notification.go b/internal/rpc/relation/notification.go similarity index 99% rename from internal/rpc/friend/notification.go rename to internal/rpc/relation/notification.go index 5fb34577fc..83c5d2ca9d 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/relation/notification.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/sync.go b/internal/rpc/relation/sync.go similarity index 99% rename from internal/rpc/friend/sync.go rename to internal/rpc/relation/sync.go index 902cc73037..0ad94fe825 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/relation/sync.go @@ -1,4 +1,4 @@ -package friend +package relation import ( "context" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 1e534437df..a6952bd6db 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,7 +17,7 @@ package user import ( "context" "errors" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -54,7 +54,7 @@ import ( type userServer struct { online cache.OnlineCache db controller.UserDatabase - friendNotificationSender *friend.FriendNotificationSender + friendNotificationSender *relation.FriendNotificationSender userNotificationSender *UserNotificationSender friendRpcClient *rpcclient.FriendRpcClient groupRpcClient *rpcclient.GroupRpcClient @@ -105,7 +105,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi RegisterCenter: client, friendRpcClient: &friendRpcClient, groupRpcClient: &groupRpcClient, - friendNotificationSender: friend.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, friend.WithDBFunc(database.FindWithError)), + friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, relation.WithDBFunc(database.FindWithError)), userNotificationSender: NewUserNotificationSender(config, &msgRpcClient, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 1bc9e6c543..a564facd06 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -17,7 +17,7 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -26,23 +26,23 @@ import ( type FriendRpcCmd struct { *RootCmd - ctx context.Context - configMap map[string]any - friendConfig *friend.Config + ctx context.Context + configMap map[string]any + relationConfig *relation.Config } func NewFriendRpcCmd() *FriendRpcCmd { - var friendConfig friend.Config - ret := &FriendRpcCmd{friendConfig: &friendConfig} + var relationConfig relation.Config + ret := &FriendRpcCmd{relationConfig: &relationConfig} ret.configMap = map[string]any{ - OpenIMRPCFriendCfgFileName: &friendConfig.RpcConfig, - RedisConfigFileName: &friendConfig.RedisConfig, - MongodbConfigFileName: &friendConfig.MongodbConfig, - ShareFileName: &friendConfig.Share, - NotificationFileName: &friendConfig.NotificationConfig, - WebhooksConfigFileName: &friendConfig.WebhooksConfig, - LocalCacheConfigFileName: &friendConfig.LocalCacheConfig, - DiscoveryConfigFilename: &friendConfig.Discovery, + OpenIMRPCFriendCfgFileName: &relationConfig.RpcConfig, + RedisConfigFileName: &relationConfig.RedisConfig, + MongodbConfigFileName: &relationConfig.MongodbConfig, + ShareFileName: &relationConfig.Share, + NotificationFileName: &relationConfig.NotificationConfig, + WebhooksConfigFileName: &relationConfig.WebhooksConfig, + LocalCacheConfigFileName: &relationConfig.LocalCacheConfig, + DiscoveryConfigFilename: &relationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -57,7 +57,7 @@ func (a *FriendRpcCmd) Exec() error { } func (a *FriendRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.friendConfig.Discovery, &a.friendConfig.RpcConfig.Prometheus, a.friendConfig.RpcConfig.RPC.ListenIP, - a.friendConfig.RpcConfig.RPC.RegisterIP, a.friendConfig.RpcConfig.RPC.Ports, - a.Index(), a.friendConfig.Share.RpcRegisterName.Friend, &a.friendConfig.Share, a.friendConfig, friend.Start) + return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, + a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.Ports, + a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } From 375f63a447dcbbdafbeade31b5a53b4b780fd24d Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:11:17 +0800 Subject: [PATCH 47/91] fix: online status renewal (#2468) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs * add some logs * fix: onlineUserIDs * add logs * test * test * test * test --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- internal/msggateway/online.go | 9 +++++++-- internal/msggateway/user_map.go | 11 ++++++++--- internal/push/push_handler.go | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index b50608f93c..27b4544aaa 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -4,13 +4,16 @@ import ( "context" "crypto/md5" "encoding/binary" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" pbuser "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "math/rand" + "os" "strconv" + "sync/atomic" "time" ) @@ -78,8 +81,10 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { } } - opIdCtx := mcontext.SetOperationID(context.Background(), "r"+strconv.FormatUint(rNum, 10)) + var count atomic.Int64 + operationIDPrefix := fmt.Sprintf("p_%d_", os.Getpid()) doRequest := func(req *pbuser.SetUserOnlineStatusReq) { + opIdCtx := mcontext.SetOperationID(context.Background(), operationIDPrefix+strconv.FormatInt(count.Add(1), 10)) ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) defer cancel() if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { @@ -102,7 +107,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case now := <-renewalTicker.C: deadline := now.Add(-cachekey.OnlineExpire / 3) users := ws.clients.GetAllUserStatus(deadline, now) - log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users)) + log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users), "users", users) pushUserState(users...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index bd1f197281..36cab4ed75 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,6 +1,10 @@ package msggateway import ( + "context" + "fmt" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "sync" "time" @@ -117,6 +121,7 @@ func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { } func (u *userMap) Set(userID string, client *Client) { + log.ZDebug(context.Background(), "userMap Set", "userID", userID, "platformID", client.PlatformID, "platform", constant.PlatformIDToName(client.PlatformID), "pointer", fmt.Sprintf("%p", client)) u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] @@ -162,12 +167,12 @@ func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser return true } -func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) (result []UserState) { u.lock.RLock() defer u.lock.RUnlock() - result := make([]UserState, 0, len(u.data)) + result = make([]UserState, 0, len(u.data)) for userID, userPlatform := range u.data { - if userPlatform.Time.Before(deadline) { + if deadline.Before(userPlatform.Time) { continue } userPlatform.Time = nowtime diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index c3b27372b3..249622a592 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -202,7 +202,7 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { var err error - result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, onlineUserIDs) if err != nil { return nil, err } From 51e170ade17e898979c6e616fbd89b9cca0b56d3 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 1 Aug 2024 16:12:41 +0800 Subject: [PATCH 48/91] feat: implement scheduled destruct msgs feature in cron task. (#2466) * update protocol in go mod. * add debug log in writePongMsg. * update log level. * add Warn log in writePongMsg. * add debug log. * feat: update webhookBeforeMemberJoinGroup to batch method. * feat: update version field implement. * update webhook implement contents. * update method field and contents. * update callbackCommand field. * fix: add correct fields. * update struct tags. * refactor: rename friend module to relation. * feat: implement scheduled destruct msgs feature in cron task. * update log contents. * update func name and comments. * update waitgroup to errgroup. * update errgroup wait. * remove unnecessary contents. * update clearMsg logic. --- go.mod | 2 +- go.sum | 4 +- internal/rpc/conversation/conversaion.go | 89 +++++++++++++++++++----- internal/rpc/msg/clear.go | 85 +++++++++++++++++----- internal/rpc/msg/server.go | 5 ++ internal/tools/cron_task.go | 65 +++++++++++++---- pkg/common/convert/conversation.go | 4 +- pkg/rpcclient/conversation.go | 10 ++- 8 files changed, 210 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 1a1cf36d2d..49254097a4 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.47 + github.com/openimsdk/protocol v0.0.69-alpha.50 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 815afe8d21..ac525b2db6 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.47 h1:WEpU7dHSzcpiyPoUkgSt1mC9HfQ6xSDNNZf4KWbZiFI= -github.com/openimsdk/protocol v0.0.69-alpha.47/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.50 h1:4r6vY9LsjFrR8AAwORFhijOGmq2vzDH3XTX4wBiw+2M= +github.com/openimsdk/protocol v0.0.69-alpha.50/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 3047c376bb..4cf20f919c 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -16,13 +16,16 @@ package conversation import ( "context" + "sort" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/db/redisutil" - "sort" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -40,10 +43,11 @@ import ( ) type conversationServer struct { - msgRpcClient *rpcclient.MessageRpcClient - user *rpcclient.UserRpcClient - groupRpcClient *rpcclient.GroupRpcClient - conversationDatabase controller.ConversationDatabase + msgRpcClient *rpcclient.MessageRpcClient + user *rpcclient.UserRpcClient + groupRpcClient *rpcclient.GroupRpcClient + conversationDatabase controller.ConversationDatabase + conversationNotificationSender *ConversationNotificationSender config *Config } @@ -204,11 +208,11 @@ func (c *conversationServer) getConversations(ctx context.Context, ownerUserID s } func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { - var conversation tablerelation.Conversation + var conversation dbModel.Conversation if err := datautil.CopyStructFields(&conversation, req.Conversation); err != nil { return nil, err } - err := c.conversationDatabase.SetUserConversations(ctx, req.Conversation.OwnerUserID, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.SetUserConversations(ctx, req.Conversation.OwnerUserID, []*dbModel.Conversation{&conversation}) if err != nil { return nil, err } @@ -232,7 +236,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } } var unequal int - var conv tablerelation.Conversation + var conv dbModel.Conversation if len(req.UserIDs) == 1 { cs, err := c.conversationDatabase.FindConversations(ctx, req.UserIDs[0], []string{req.Conversation.ConversationID}) if err != nil { @@ -243,7 +247,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } conv = *cs[0] } - var conversation tablerelation.Conversation + var conversation dbModel.Conversation conversation.ConversationID = req.Conversation.ConversationID conversation.ConversationType = req.Conversation.ConversationType conversation.UserID = req.Conversation.UserID @@ -292,7 +296,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { - var conversations []*tablerelation.Conversation + var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { conversation2 := conversation conversation2.OwnerUserID = ownerUserID @@ -340,12 +344,12 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, ) (*pbconversation.CreateSingleChatConversationsResp, error) { switch req.ConversationType { case constant.SingleChatType: - var conversation tablerelation.Conversation + var conversation dbModel.Conversation conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.SendID conversation.UserID = req.RecvID - err := c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation", conversation) } @@ -353,17 +357,17 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, conversation2 := conversation conversation2.OwnerUserID = req.RecvID conversation2.UserID = req.SendID - err = c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation2}) + err = c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation2}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } case constant.NotificationChatType: - var conversation tablerelation.Conversation + var conversation dbModel.Conversation conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.RecvID conversation.UserID = req.SendID - err := c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } @@ -584,6 +588,9 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv if req.MaxSeq != nil { m["max_seq"] = req.MaxSeq.Value } + if req.LatestMsgDestructTime != nil { + m["latest_msg_destruct_time"] = time.UnixMilli(req.LatestMsgDestructTime.Value) + } if len(m) > 0 { if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.UserIDs, req.ConversationID, m); err != nil { return nil, err @@ -602,3 +609,53 @@ func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbco Conversations: convert.ConversationsDB2Pb(conversations), }, nil } + +func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Context, _ *pbconversation.GetConversationsNeedDestructMsgsReq) (*pbconversation.GetConversationsNeedDestructMsgsResp, error) { + num, err := c.conversationDatabase.GetAllConversationIDsNumber(ctx) + if err != nil { + log.ZError(ctx, "GetAllConversationIDsNumber failed", err) + return nil, err + } + const batchNum = 100 + + if num == 0 { + return nil, errs.New("Need Destruct Msg is nil").Wrap() + } + + maxPage := (num + batchNum - 1) / batchNum + + temp := make([]*model.Conversation, 0, maxPage*batchNum) + + for pageNumber := 0; pageNumber < int(maxPage); pageNumber++ { + pagination := &sdkws.RequestPagination{ + PageNumber: int32(pageNumber), + ShowNumber: batchNum, + } + + conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) + if err != nil { + log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) + continue + } + + log.ZDebug(ctx, "PageConversationIDs success", "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) + if len(conversationIDs) == 0 { + continue + } + + conversations, err := c.conversationDatabase.GetConversationsByConversationID(ctx, conversationIDs) + if err != nil { + log.ZError(ctx, "GetConversationsByConversationID failed", err, "conversationIDs", conversationIDs) + continue + } + + for _, conversation := range conversations { + if conversation.IsMsgDestruct && conversation.MsgDestructTime != 0 && ((time.Now().UnixMilli() > (conversation.MsgDestructTime + conversation.LatestMsgDestructTime.UnixMilli() + 8*60*60)) || // 8*60*60 is UTC+8 + conversation.LatestMsgDestructTime.IsZero()) { + temp = append(temp, conversation) + } + } + } + + return &pbconversation.GetConversationsNeedDestructMsgsResp{Conversations: convert.ConversationsDB2Pb(temp)}, nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 774eae32cd..6be551eada 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,16 +2,22 @@ package msg import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "strings" - "time" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/idutil" + "github.com/openimsdk/tools/utils/stringutil" + "golang.org/x/sync/errgroup" ) +// hard delete in Database. func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -25,18 +31,6 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. start = time.Now() ) clearMsg := func(ctx context.Context) (bool, error) { - conversationSeqs := make(map[string]struct{}) - defer func() { - req := &conversation.UpdateConversationReq{ - MsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), - } - for conversationID := range conversationSeqs { - req.ConversationID = conversationID - if err := m.Conversation.UpdateConversations(ctx, req); err != nil { - log.ZError(ctx, "update conversation max seq failed", err, "conversationID", conversationID, "msgDestructTime", req.MsgDestructTime) - } - } - }() msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, 100) if err != nil { return false, err @@ -44,6 +38,7 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. if len(msgs) == 0 { return false, nil } + for _, msg := range msgs { index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) if err != nil { @@ -52,15 +47,14 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. if len(index) == 0 { return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") } + docNum++ msgNum += len(index) - conversationID := msg.DocID[:strings.LastIndex(msg.DocID, ":")] - if _, ok := conversationSeqs[conversationID]; !ok { - conversationSeqs[conversationID] = struct{}{} - } } + return true, nil } + for { keep, err := clearMsg(ctx) if err != nil { @@ -71,7 +65,60 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. log.ZInfo(ctx, "clear msg success", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) break } + log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) } return &msg.ClearMsgResp{}, nil } + +// soft delete for self +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { + temp := convert.ConversationsPb2DB(req.Conversations) + + batchNum := 100 + + errg, _ := errgroup.WithContext(ctx) + errg.SetLimit(100) + + for i := 0; i < len(temp); i += batchNum { + batch := temp[i:min(i+batchNum, len(temp))] + + errg.Go(func() error { + for _, conversation := range batch { + handleCtx := mcontext.NewCtx(stringutil.GetSelfFuncName() + "-" + idutil.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID) + log.ZDebug(handleCtx, "User MsgsDestruct", + "conversationID", conversation.ConversationID, + "ownerUserID", conversation.OwnerUserID, + "msgDestructTime", conversation.MsgDestructTime, + "lastMsgDestructTime", conversation.LatestMsgDestructTime) + + seqs, err := m.MsgDatabase.UserMsgsDestruct(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) + if err != nil { + log.ZError(handleCtx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) + continue + } + + if len(seqs) > 0 { + if err := m.Conversation.UpdateConversation(handleCtx, + &pbconversation.UpdateConversationReq{ + UserIDs: []string{conversation.OwnerUserID}, + ConversationID: conversation.ConversationID, + LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli())}); err != nil { + log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) + continue + } + + // if you need Notify SDK client userseq is update. + // m.msgNotificationSender.UserDeleteMsgsNotification(handleCtx, conversation.OwnerUserID, conversation.ConversationID, seqs) + } + } + return nil + }) + } + + if err := errg.Wait(); err != nil { + return nil, err + } + + return nil, nil +} diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index de0f698ea3..91f41f1b11 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -50,6 +51,7 @@ type ( ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data. Handlers MessageInterceptorChain // Chain of handlers for processing messages. notificationSender *rpcclient.NotificationSender // RPC client for sending notifications. + msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. config *Config // Global configuration settings. webhookClient *webhook.Client } @@ -117,7 +119,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } s.notificationSender = rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithLocalSendMsg(s.SendMsg)) + s.msgNotificationSender = NewMsgNotificationSender(config, rpcclient.WithLocalSendMsg(s.SendMsg)) + msg.RegisterMsgServer(server, s) + return nil } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 1ef4943cd6..b1d59800ce 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -17,16 +17,19 @@ package tools import ( "context" "fmt" + "os" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "os" - "time" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -50,34 +53,69 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) - conn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) + + msgConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) if err != nil { return err } - cli := msg.NewMsgClient(conn) + + thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return err + } + + conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) + if err != nil { + return err + } + + msgClient := msg.NewMsgClient(msgConn) + conversationClient := pbconversation.NewConversationClient(conversationConn) + thirdClient := third.NewThirdClient(thirdConn) + crontab := cron.New() - clearFunc := func() { + + // scheduled hard delete outdated Msgs in specific time. + clearMsgFunc := func() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - if _, err := cli.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { + if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { return errs.Wrap(err) } - tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) - if err != nil { - return err + // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. + msgDestructFunc := func() { + now := time.Now() + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) + log.ZInfo(ctx, "msg destruct cron start", "now", now) + + conversations, err := conversationClient.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) + if err != nil { + log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) + return + } else { + _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Destruct Msgs failed.", err) + return + } + } + log.ZInfo(ctx, "msg destruct cron task completed", "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, msgDestructFunc); err != nil { + return errs.Wrap(err) } - thirdClient := third.NewThirdClient(tConn) - deleteFunc := func() { + // scheduled delete outdated file Objects and their datas in specific time. + deleteObjectFunc := func() { now := time.Now() deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) @@ -88,9 +126,10 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { return errs.Wrap(err) } + log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() diff --git a/pkg/common/convert/conversation.go b/pkg/common/convert/conversation.go index a76d7d9f6d..9389b02524 100644 --- a/pkg/common/convert/conversation.go +++ b/pkg/common/convert/conversation.go @@ -22,7 +22,7 @@ import ( func ConversationDB2Pb(conversationDB *model.Conversation) *conversation.Conversation { conversationPB := &conversation.Conversation{} - conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix() + conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.UnixMilli() if err := datautil.CopyStructFields(conversationPB, conversationDB); err != nil { return nil } @@ -35,7 +35,7 @@ func ConversationsDB2Pb(conversationsDB []*model.Conversation) (conversationsPB if err := datautil.CopyStructFields(conversationPB, conversationDB); err != nil { continue } - conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix() + conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.UnixMilli() conversationsPB = append(conversationsPB, conversationPB) } return conversationsPB diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go index e078f432b3..8f95f86a6c 100644 --- a/pkg/rpcclient/conversation.go +++ b/pkg/rpcclient/conversation.go @@ -82,7 +82,7 @@ func (c *ConversationRpcClient) SetConversations(ctx context.Context, userIDs [] return err } -func (c *ConversationRpcClient) UpdateConversations(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { +func (c *ConversationRpcClient) UpdateConversation(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { _, err := c.Client.UpdateConversation(ctx, conversation) return err } @@ -146,3 +146,11 @@ func (c *ConversationRpcClient) GetConversationNotReceiveMessageUserIDs(ctx cont } return resp.UserIDs, nil } + +func (c *ConversationRpcClient) GetConversationsNeedDestructMsgs(ctx context.Context) ([]*pbconversation.Conversation, error) { + resp, err := c.Client.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) + if err != nil { + return nil, err + } + return resp.Conversations, nil +} From 84049a1f55632903fb51afcd27967931384771e8 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:58:04 +0800 Subject: [PATCH 49/91] fix: the local cache obtained can be modified (#2473) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs * add some logs * fix: onlineUserIDs * add logs * test * test * test * test * add log * feat: solve the problem that modifying the cached data affects other * feat: solve the problem that modifying the cached data affects other * feat: search messages to filter out notifications --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- internal/msggateway/user_map.go | 5 --- pkg/common/storage/database/mgo/msg.go | 11 ++++++ pkg/rpccache/common.go | 43 ++++++++++++++++++++ pkg/rpccache/conversation.go | 55 ++++++++++++++++++-------- pkg/rpccache/friend.go | 52 ++++++++++++++++-------- pkg/rpccache/group.go | 39 ++++++++++-------- pkg/rpccache/online.go | 14 ++++++- pkg/rpccache/user.go | 47 ++++++++++------------ 8 files changed, 183 insertions(+), 83 deletions(-) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 36cab4ed75..5baa4f9012 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,10 +1,6 @@ package msggateway import ( - "context" - "fmt" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "sync" "time" @@ -121,7 +117,6 @@ func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { } func (u *userMap) Set(userID string, client *Client) { - log.ZDebug(context.Background(), "userMap Set", "userID", userID, "platformID", client.PlatformID, "platform", constant.PlatformIDToName(client.PlatformID), "pointer", fmt.Sprintf("%p", client)) u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 03f47c5033..7dc308a7c4 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -377,8 +377,19 @@ func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID prim if !nextID.IsZero() { pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) } + coarseFilter := bson.M{ + "$or": bson.A{ + bson.M{ + "doc_id": primitive.Regex{Pattern: "^sg_"}, + }, + bson.M{ + "doc_id": primitive.Regex{Pattern: "^si_"}, + }, + }, + } pipeline = append(pipeline, bson.M{"$sort": bson.M{"_id": 1}}, + bson.M{"$match": coarseFilter}, bson.M{"$match": filter}, bson.M{"$limit": limit}, bson.M{ diff --git a/pkg/rpccache/common.go b/pkg/rpccache/common.go index 5ed007edc4..15b3a8e094 100644 --- a/pkg/rpccache/common.go +++ b/pkg/rpccache/common.go @@ -14,6 +14,11 @@ package rpccache +import ( + "github.com/openimsdk/tools/errs" + "google.golang.org/protobuf/proto" +) + func newListMap[V comparable](values []V, err error) (*listMap[V], error) { if err != nil { return nil, err @@ -32,3 +37,41 @@ type listMap[V comparable] struct { List []V Map map[V]struct{} } + +func respProtoMarshal(resp proto.Message, err error) ([]byte, error) { + if err != nil { + return nil, err + } + return proto.Marshal(resp) +} + +func cacheUnmarshal[V any](resp []byte, err error) (*V, error) { + if err != nil { + return nil, err + } + var val V + if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil { + return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error") + } + return &val, nil +} + +type cacheProto[V any] struct{} + +func (cacheProto[V]) Marshal(resp *V, err error) ([]byte, error) { + if err != nil { + return nil, err + } + return proto.Marshal(any(resp).(proto.Message)) +} + +func (cacheProto[V]) Unmarshal(resp []byte, err error) (*V, error) { + if err != nil { + return nil, err + } + var val V + if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil { + return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error") + } + return &val, nil +} diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 0109f1b1dd..2a62c7bbd5 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -23,6 +23,7 @@ import ( pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "golang.org/x/sync/errgroup" ) @@ -36,7 +37,7 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &ConversationLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -52,21 +53,30 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach type ConversationLocalCache struct { client rpcclient.ConversationRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs req", "ownerUserID", ownerUserID) + resp, err := c.getConversationIDs(ctx, ownerUserID) + if err != nil { + return nil, err + } + return resp.ConversationIDs, nil +} + +func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconversation.GetConversationIDsResp, err error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs req", "ownerUserID", ownerUserID) defer func() { if err == nil { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs return", "value", val) + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs return", "ownerUserID", ownerUserID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversationIDs return", err) + log.ZError(ctx, "ConversationLocalCache getConversationIDs return", err, "ownerUserID", ownerUserID) } }() - return localcache.AnyValue[[]string](c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) (any, error) { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs rpc", "ownerUserID", ownerUserID) - return c.client.GetConversationIDs(ctx, ownerUserID) + var cache cacheProto[pbconversation.GetConversationIDsResp] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs rpc", "ownerUserID", ownerUserID) + return cache.Marshal(c.client.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID})) })) } @@ -74,14 +84,15 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID) defer func() { if err == nil { - log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "value", val) + log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "userID", userID, "conversationID", conversationID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversation return", err) + log.ZError(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() - return localcache.AnyValue[*pbconversation.Conversation](c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) (any, error) { + var cache cacheProto[pbconversation.Conversation] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID) - return c.client.GetConversation(ctx, userID, conversationID) + return cache.Marshal(c.client.GetConversation(ctx, userID, conversationID)) })) } @@ -126,9 +137,19 @@ func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUser return conversations, nil } -func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (*listMap[string], error) { - return localcache.AnyValue[*listMap[string]](c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) (any, error) { - return newListMap(c.client.GetConversationNotReceiveMessageUserIDs(ctx, conversationID)) +func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconversation.GetConversationNotReceiveMessageUserIDsResp, err error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs req", "conversationID", conversationID) + defer func() { + if err == nil { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", "conversationID", conversationID, "value", val) + } else { + log.ZError(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", err, "conversationID", conversationID) + } + }() + var cache cacheProto[pbconversation.GetConversationNotReceiveMessageUserIDsResp] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "conversationID", conversationID) + return cache.Marshal(c.client.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) })) } @@ -137,7 +158,7 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDs(ctx con if err != nil { return nil, err } - return res.List, nil + return res.UserIDs, nil } func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx context.Context, conversationID string) (map[string]struct{}, error) { @@ -145,5 +166,5 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx c if err != nil { return nil, err } - return res.Map, nil + return datautil.SliceSet(res.UserIDs), nil } diff --git a/pkg/rpccache/friend.go b/pkg/rpccache/friend.go index a5cee2567c..dca3b4c97c 100644 --- a/pkg/rpccache/friend.go +++ b/pkg/rpccache/friend.go @@ -16,7 +16,8 @@ package rpccache import ( "context" - cachekey2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -30,7 +31,7 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &FriendLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -46,36 +47,55 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo type FriendLocalCache struct { client rpcclient.FriendRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) { - log.ZDebug(ctx, "FriendLocalCache IsFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID) + res, err := f.isFriend(ctx, possibleFriendUserID, userID) + if err != nil { + return false, err + } + return res.InUser1Friends, nil +} + +func (f *FriendLocalCache) isFriend(ctx context.Context, possibleFriendUserID, userID string) (val *relation.IsFriendResp, err error) { + log.ZDebug(ctx, "FriendLocalCache isFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "FriendLocalCache IsFriend return", "value", val) + log.ZDebug(ctx, "FriendLocalCache isFriend return", "possibleFriendUserID", possibleFriendUserID, "userID", userID, "value", val) } else { - log.ZError(ctx, "FriendLocalCache IsFriend return", err) + log.ZError(ctx, "FriendLocalCache isFriend return", err, "possibleFriendUserID", possibleFriendUserID, "userID", userID) } }() - return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) (any, error) { - log.ZDebug(ctx, "FriendLocalCache IsFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) - return f.client.IsFriend(ctx, possibleFriendUserID, userID) - }, cachekey2.GetFriendIDsKey(possibleFriendUserID))) + var cache cacheProto[relation.IsFriendResp] + return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "FriendLocalCache isFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) + return cache.Marshal(f.client.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) + }, cachekey.GetFriendIDsKey(possibleFriendUserID))) } // IsBlack possibleBlackUserID selfUserID. func (f *FriendLocalCache) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (val bool, err error) { - log.ZDebug(ctx, "FriendLocalCache IsBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID) + res, err := f.isBlack(ctx, possibleBlackUserID, userID) + if err != nil { + return false, err + } + return res.InUser2Blacks, nil +} + +// IsBlack possibleBlackUserID selfUserID. +func (f *FriendLocalCache) isBlack(ctx context.Context, possibleBlackUserID, userID string) (val *relation.IsBlackResp, err error) { + log.ZDebug(ctx, "FriendLocalCache isBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "FriendLocalCache IsBlack return", "value", val) + log.ZDebug(ctx, "FriendLocalCache isBlack return", "possibleBlackUserID", possibleBlackUserID, "userID", userID, "value", val) } else { - log.ZError(ctx, "FriendLocalCache IsBlack return", err) + log.ZError(ctx, "FriendLocalCache isBlack return", err, "possibleBlackUserID", possibleBlackUserID, "userID", userID) } }() - return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) (any, error) { + var cache cacheProto[relation.IsBlackResp] + return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID) - return f.client.IsBlack(ctx, possibleBlackUserID, userID) - }, cachekey2.GetBlackIDsKey(userID))) + return cache.Marshal(f.client.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) + }, cachekey.GetBlackIDsKey(userID))) } diff --git a/pkg/rpccache/group.go b/pkg/rpccache/group.go index 55e1438be3..b2d852fc5f 100644 --- a/pkg/rpccache/group.go +++ b/pkg/rpccache/group.go @@ -17,6 +17,8 @@ package rpccache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -32,7 +34,7 @@ func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.Loca log.ZDebug(context.Background(), "GroupLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &GroupLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -48,21 +50,22 @@ func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.Loca type GroupLocalCache struct { client rpcclient.GroupRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } -func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *listMap[string], err error) { +func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *group.GetGroupMemberUserIDsResp, err error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs req", "groupID", groupID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs return", "value", val) + log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs return", "groupID", groupID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache getGroupMemberIDs return", err) + log.ZError(ctx, "GroupLocalCache getGroupMemberIDs return", err, "groupID", groupID) } }() - return localcache.AnyValue[*listMap[string]](g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) (any, error) { + var cache cacheProto[group.GetGroupMemberUserIDsResp] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs rpc", "groupID", groupID) - return newListMap(g.client.GetGroupMemberIDs(ctx, groupID)) + return cache.Marshal(g.client.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) })) } @@ -70,14 +73,15 @@ func (g *GroupLocalCache) GetGroupMember(ctx context.Context, groupID, userID st log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val) + log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "groupID", groupID, "userID", userID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err) + log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err, "groupID", groupID, "userID", userID) } }() - return localcache.AnyValue[*sdkws.GroupMemberFullInfo](g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.GroupMemberFullInfo] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID, "userID", userID) - return g.client.GetGroupMemberCache(ctx, groupID, userID) + return cache.Marshal(g.client.GetGroupMemberCache(ctx, groupID, userID)) })) } @@ -85,14 +89,15 @@ func (g *GroupLocalCache) GetGroupInfo(ctx context.Context, groupID string) (val log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val) + log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "groupID", groupID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err) + log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err, "groupID", groupID) } }() - return localcache.AnyValue[*sdkws.GroupInfo](g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.GroupInfo] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID) - return g.client.GetGroupInfoCache(ctx, groupID) + return cache.Marshal(g.client.GetGroupInfoCache(ctx, groupID)) })) } @@ -101,7 +106,7 @@ func (g *GroupLocalCache) GetGroupMemberIDs(ctx context.Context, groupID string) if err != nil { return nil, err } - return res.List, nil + return res.UserIDs, nil } func (g *GroupLocalCache) GetGroupMemberIDMap(ctx context.Context, groupID string) (map[string]struct{}, error) { @@ -109,7 +114,7 @@ func (g *GroupLocalCache) GetGroupMemberIDMap(ctx context.Context, groupID strin if err != nil { return nil, err } - return res.Map, nil + return datautil.SliceSet(res.UserIDs), nil } func (g *GroupLocalCache) GetGroupInfos(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 13578a7df6..2ffa1f1577 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -47,7 +47,7 @@ type OnlineCache struct { local lru.LRU[string, []int32] } -func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { +func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { platformIDs, err := o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) @@ -59,8 +59,18 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } +func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + platformIDs, err := o.getUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + tmp := make([]int32, len(platformIDs)) + copy(tmp, platformIDs) + return platformIDs, nil +} + func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { - platformIDs, err := o.GetUserOnlinePlatform(ctx, userID) + platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { return false, err } diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 6126f5891c..7c676f30a1 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -16,12 +16,12 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -32,7 +32,7 @@ func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalC log.ZDebug(context.Background(), "UserLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &UserLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -48,7 +48,7 @@ func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalC type UserLocalCache struct { client rpcclient.UserRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *sdkws.UserInfo, err error) { @@ -60,24 +60,34 @@ func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *s log.ZError(ctx, "UserLocalCache GetUserInfo return", err) } }() - return localcache.AnyValue[*sdkws.UserInfo](u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.UserInfo] + return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserInfo rpc", "userID", userID) - return u.client.GetUserInfo(ctx, userID) + return cache.Marshal(u.client.GetUserInfo(ctx, userID)) })) } func (u *UserLocalCache) GetUserGlobalMsgRecvOpt(ctx context.Context, userID string) (val int32, err error) { - log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt req", "userID", userID) + resp, err := u.getUserGlobalMsgRecvOpt(ctx, userID) + if err != nil { + return 0, err + } + return resp.GlobalRecvMsgOpt, nil +} + +func (u *UserLocalCache) getUserGlobalMsgRecvOpt(ctx context.Context, userID string) (val *user.GetGlobalRecvMessageOptResp, err error) { + log.ZDebug(ctx, "UserLocalCache getUserGlobalMsgRecvOpt req", "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", "value", val) + log.ZDebug(ctx, "UserLocalCache getUserGlobalMsgRecvOpt return", "value", val) } else { - log.ZError(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", err) + log.ZError(ctx, "UserLocalCache getUserGlobalMsgRecvOpt return", err) } }() - return localcache.AnyValue[int32](u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) (any, error) { + var cache cacheProto[user.GetGlobalRecvMessageOptResp] + return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) - return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) + return cache.Marshal(u.client.Client.GetGlobalRecvMessageOpt(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) })) } @@ -110,18 +120,3 @@ func (u *UserLocalCache) GetUsersInfoMap(ctx context.Context, userIDs []string) } return users, nil } - -//func (u *UserLocalCache) GetUserOnlinePlatform(ctx context.Context, userID string) (val []int32, err error) { -// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform req", "userID", userID) -// defer func() { -// if err == nil { -// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform return", "value", val) -// } else { -// log.ZError(ctx, "UserLocalCache GetUserOnlinePlatform return", err) -// } -// }() -// return localcache.AnyValue[[]int32](u.local.Get(ctx, cachekey.GetOnlineKey(userID), func(ctx context.Context) (any, error) { -// log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) -// return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) -// })) -//} From c45967079eb20d4c756b7476e4d5a28c51b0e30f Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 2 Aug 2024 18:30:17 +0800 Subject: [PATCH 50/91] feat: update go mod pkg to latest. (#2475) --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 49254097a4..d1873d3528 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.50 - github.com/openimsdk/tools v0.0.49-alpha.55 + github.com/openimsdk/protocol v0.0.69 + github.com/openimsdk/tools v0.0.49 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -74,6 +74,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chai2010/webp v1.1.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/coreos/go-semver v0.3.0 // indirect diff --git a/go.sum b/go.sum index ac525b2db6..95e16f81af 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk= +github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -319,10 +321,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.50 h1:4r6vY9LsjFrR8AAwORFhijOGmq2vzDH3XTX4wBiw+2M= -github.com/openimsdk/protocol v0.0.69-alpha.50/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= -github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/protocol v0.0.69 h1:dVi8meSg8kmUzSH1XQab4MjihqKkkcCAmt1BYXPJuXo= +github.com/openimsdk/protocol v0.0.69/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49 h1:yILTgOCqxlqJMc889fE99E5ZGa70v/E3hkCSeTnWl3s= +github.com/openimsdk/tools v0.0.49/go.mod h1:oiSQU5Z6fzjxKFjbqDHImD8EmCIwClU1Rkur1sK12Po= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 840ddc15fe12742656fdaba80e7e19b63f27b845 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 2 Aug 2024 20:13:13 +0800 Subject: [PATCH 51/91] chore: revert tool pkg version. (#2476) * feat: update go mod pkg to latest. * revert tool pkg version. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d1873d3528..fba1499fe1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69 - github.com/openimsdk/tools v0.0.49 + github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 95e16f81af..1a8e1d76d8 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69 h1:dVi8meSg8kmUzSH1XQab4MjihqKkkcCAmt1BYXPJuXo= github.com/openimsdk/protocol v0.0.69/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49 h1:yILTgOCqxlqJMc889fE99E5ZGa70v/E3hkCSeTnWl3s= -github.com/openimsdk/tools v0.0.49/go.mod h1:oiSQU5Z6fzjxKFjbqDHImD8EmCIwClU1Rkur1sK12Po= +github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= +github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 4c8fcac9c79fb297323baeb75b786b24dad2b04b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:14:22 +0800 Subject: [PATCH 52/91] feat: update grafana template (#2484) --- config/grafana-template/Demo.json | 1768 +++++++++++++++-------------- 1 file changed, 929 insertions(+), 839 deletions(-) diff --git a/config/grafana-template/Demo.json b/config/grafana-template/Demo.json index dbb11fbf31..c4668917f4 100644 --- a/config/grafana-template/Demo.json +++ b/config/grafana-template/Demo.json @@ -1,4 +1,35 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -18,7 +49,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": null, "links": [], "liveNow": false, "panels": [ @@ -35,7 +66,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "Is the service up.", "fieldConfig": { @@ -84,8 +115,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -121,7 +151,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -141,7 +171,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of online users and login users within the time frame.", "fieldConfig": { @@ -190,8 +220,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -244,7 +273,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -260,7 +289,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "increase(user_login_total[$time])", @@ -277,7 +306,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of register users within the time frame.", "fieldConfig": { @@ -326,8 +355,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -380,7 +408,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -400,7 +428,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of chat msg success.", "fieldConfig": { @@ -449,8 +477,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -486,7 +513,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -502,7 +529,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "increase(group_chat_msg_process_success_total[$time])", @@ -519,7 +546,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of chat msg failed .", "fieldConfig": { @@ -568,8 +595,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -637,7 +663,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -653,7 +679,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "increase(group_chat_msg_process_failed_total[$time])", @@ -670,7 +696,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of msg failed offline pushed.", "fieldConfig": { @@ -719,8 +745,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -773,7 +798,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -793,7 +818,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of failed set seq.", "fieldConfig": { @@ -842,8 +867,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -896,7 +920,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -916,7 +940,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of successfully inserted messages.", "fieldConfig": { @@ -965,8 +989,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1002,7 +1025,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1018,7 +1041,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "increase(msg_insert_mongo_success_total[$time])", @@ -1035,7 +1058,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of failed insertion messages.", "fieldConfig": { @@ -1084,8 +1107,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1121,7 +1143,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1137,7 +1159,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "increase(msg_insert_mongo_failed_total[$time])", @@ -1168,7 +1190,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of call of all API.", "fieldConfig": { @@ -1217,8 +1239,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1254,7 +1275,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1274,7 +1295,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of call of all API within the time frame.", "fieldConfig": { @@ -1323,8 +1344,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1385,7 +1405,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1405,7 +1425,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of err return of API.", "fieldConfig": { @@ -1454,8 +1474,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1491,7 +1510,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1511,7 +1530,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of err return of API with err code.", "fieldConfig": { @@ -1560,8 +1579,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1597,7 +1615,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1617,7 +1635,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the qps of API.", "fieldConfig": { @@ -1666,8 +1684,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1719,7 +1736,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1739,7 +1756,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of err return of API within the time frame.", "fieldConfig": { @@ -1788,8 +1805,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1825,7 +1841,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1845,7 +1861,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of err return of API with err code within the time frame..", "fieldConfig": { @@ -1894,8 +1910,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1931,7 +1946,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -1965,7 +1980,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of call of all RPC.", "fieldConfig": { @@ -2014,8 +2029,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2051,7 +2065,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2071,7 +2085,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the error return of RPC.", "fieldConfig": { @@ -2120,8 +2134,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2157,7 +2170,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2177,7 +2190,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the error return of RPC with code.", "fieldConfig": { @@ -2226,8 +2239,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2263,7 +2275,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2283,7 +2295,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of call of all RPC within the time frame.", "fieldConfig": { @@ -2368,7 +2380,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2388,7 +2400,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of RPC calls within the time frame, aggregated by name.", "fieldConfig": { @@ -2473,7 +2485,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2493,7 +2505,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of call of RPC within the time frame, aggregated by address.", "fieldConfig": { @@ -2578,7 +2590,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2598,7 +2610,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the error return of RPC within the time frame within the time frame.", "fieldConfig": { @@ -2683,7 +2695,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2703,7 +2715,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the error return of RPC with code within the time frame within the time frame.", "fieldConfig": { @@ -2788,7 +2800,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2822,7 +2834,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of HTTP requests.", "fieldConfig": { @@ -2871,8 +2883,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2908,7 +2919,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -2928,7 +2939,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of HTTP requests with status.", "fieldConfig": { @@ -2977,8 +2988,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3014,7 +3024,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -3034,7 +3044,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of HTTP requests within the time frame.", "fieldConfig": { @@ -3083,8 +3093,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3120,7 +3129,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -3140,7 +3149,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of HTTP requests with status within the time frame.", "fieldConfig": { @@ -3189,8 +3198,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3226,7 +3234,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -3246,7 +3254,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the qps of HTTP.", "fieldConfig": { @@ -3347,7 +3355,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -3369,7 +3377,7 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -3377,749 +3385,841 @@ "y": 4 }, "id": 6, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "panels": [], + "title": "Process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 27 + { + "color": "red", + "value": 80 + } + ] }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" - }, - { - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 9, - "libraryPanel": { - "name": "Virtual Memory bytes", - "uid": "fdriqgnk5lnnke" + { + "color": "red", + "value": 80 + } + ] }, - "title": "Virtual Memory bytes" + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the number of process virtual memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Virtual Memory bytes", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Resident Memory bytes", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Resident Memory bytes", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" } ], - "title": "Process", - "type": "row" + "title": "Resident Memory bytes", + "type": "timeseries" }, { "collapsed": true, @@ -4127,14 +4227,14 @@ "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 49 }, "id": 3, "panels": [ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", "fieldConfig": { @@ -4183,8 +4283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4220,7 +4319,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4240,7 +4339,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", "fieldConfig": { @@ -4285,8 +4384,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4322,7 +4420,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "label_replace(\r\n rate(go_gc_duration_seconds_count{job!~\"$rpcNameFilter\"}[5m]),\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", @@ -4339,7 +4437,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of goroutines.", "fieldConfig": { @@ -4388,8 +4486,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4425,7 +4522,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4445,7 +4542,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of goroutines.", "fieldConfig": { @@ -4494,8 +4591,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4531,7 +4627,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4551,7 +4647,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes allocated and still in use.", "fieldConfig": { @@ -4600,8 +4696,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4637,7 +4732,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4657,7 +4752,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes allocated and still in use.", "fieldConfig": { @@ -4706,8 +4801,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4743,7 +4837,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4763,7 +4857,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes used by the profiling bucket hash table.", "fieldConfig": { @@ -4812,8 +4906,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4849,7 +4942,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4869,7 +4962,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes used by the profiling bucket hash table.", "fieldConfig": { @@ -4918,8 +5011,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4955,7 +5047,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -4975,7 +5067,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes in use by mcache structures.", "fieldConfig": { @@ -5024,8 +5116,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5061,7 +5152,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -5081,7 +5172,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "description": "This metric represents the number of bytes in use by mcache structures.", "fieldConfig": { @@ -5130,8 +5221,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5167,7 +5257,7 @@ { "datasource": { "type": "prometheus", - "uid": "f6b25a33-8915-4220-ad0b-2c4c60eb07ab" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, @@ -5243,9 +5333,9 @@ }, { "current": { - "selected": true, - "text": "1h", - "value": "1h" + "selected": false, + "text": "5m", + "value": "5m" }, "description": "Global promQL time range.", "hide": 0, @@ -5260,7 +5350,7 @@ "value": "1m" }, { - "selected": false, + "selected": true, "text": "5m", "value": "5m" }, @@ -5270,7 +5360,7 @@ "value": "30m" }, { - "selected": true, + "selected": false, "text": "1h", "value": "1h" }, @@ -5351,6 +5441,6 @@ "timezone": "", "title": "Demo", "uid": "a506d250-b606-4702-86a7-ac6aa1d069a1", - "version": 22, + "version": 23, "weekStart": "" } \ No newline at end of file From 56e6da12c351082c1cb2de88a5075c7f0a370690 Mon Sep 17 00:00:00 2001 From: Kevin Lee <59052025+lgz5689@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:22:27 +0800 Subject: [PATCH 53/91] feat: update web front images (#2487) --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index a71332d5ee..8863f2ec1e 100644 --- a/.env +++ b/.env @@ -8,8 +8,8 @@ PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 -OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.5.1 -OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.7 +OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.0 +OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.0 #FRONT_IMAGE: use aliyun images #OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.5.1 From 9b32c630dc080fbeccdd0c6a2fd1dcc01f33c162 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:20:01 +0800 Subject: [PATCH 54/91] fix: import del cache (#2492) --- pkg/common/storage/controller/friend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 636371198a..94cb7d661d 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -192,6 +192,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, if err != nil { return err } + cache = cache.DelFriendIDs(ownerUserID).DelMaxFriendVersion(ownerUserID) if len(newMyFriendIDs) > 0 { cache = cache.DelFriendIDs(newMyFriendIDs...) cache = cache.DelFriends(ownerUserID, newMyFriendIDs).DelMaxFriendVersion(newMyFriendIDs...) From 33aa2070c47995404d0d2ad3e6c2ff3d36dc4ab9 Mon Sep 17 00:00:00 2001 From: qinguoyi <1532979219@qq.com> Date: Fri, 9 Aug 2024 10:05:12 +0800 Subject: [PATCH 55/91] fix:update recive name to same (#2493) --- pkg/rpcclient/friend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go index fd00be3292..359ed3a8b8 100644 --- a/pkg/rpcclient/friend.go +++ b/pkg/rpcclient/friend.go @@ -78,8 +78,8 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) return resp.FriendIDs, nil } -func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := b.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) +func (f *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { + r, err := f.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) if err != nil { return false, err } From 17816053f337ae833a73dd3bcebf2d8af7da66de Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 9 Aug 2024 16:53:58 +0800 Subject: [PATCH 56/91] fix: fix-validate message. (#2499) --- pkg/apistruct/msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index f4a9f884c4..dc20b5104b 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -91,7 +91,7 @@ type OANotificationElem struct { NotificationType int32 `mapstructure:"notificationType" json:"notificationType" validate:"required"` Text string `mapstructure:"text" json:"text" validate:"required"` Url string `mapstructure:"url" json:"url"` - MixType int32 `mapstructure:"mixType" json:"mixType"` + MixType int32 `mapstructure:"mixType" json:"mixType" validate:"gte=0,lte=5"` PictureElem *PictureElem `mapstructure:"pictureElem" json:"pictureElem"` SoundElem *SoundElem `mapstructure:"soundElem" json:"soundElem"` VideoElem *VideoElem `mapstructure:"videoElem" json:"videoElem"` From 7110183892d8748cf209021f211ee3895badc298 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:52:03 +0800 Subject: [PATCH 57/91] h (#2501) --- internal/rpc/msg/callback.go | 1 + pkg/callbackstruct/common.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index be58d75047..3b76c25537 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -41,6 +41,7 @@ func toCommonCallback(ctx context.Context, msg *pbchat.SendMsgReq, command strin MsgFrom: msg.MsgData.MsgFrom, ContentType: msg.MsgData.ContentType, Status: msg.MsgData.Status, + SendTime: msg.MsgData.SendTime, CreateTime: msg.MsgData.CreateTime, AtUserIDList: msg.MsgData.AtUserIDList, SenderFaceURL: msg.MsgData.SenderFaceURL, diff --git a/pkg/callbackstruct/common.go b/pkg/callbackstruct/common.go index d6714f5f20..9d6a325a8d 100644 --- a/pkg/callbackstruct/common.go +++ b/pkg/callbackstruct/common.go @@ -35,6 +35,7 @@ type CommonCallbackReq struct { MsgFrom int32 `json:"msgFrom"` ContentType int32 `json:"contentType"` Status int32 `json:"status"` + SendTime int64 `json:"sendTime"` CreateTime int64 `json:"createTime"` Content string `json:"content"` Seq uint32 `json:"seq"` From f3dfeb3bc471425ee55bfd2577aa7f2075a96d7d Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 12 Aug 2024 17:56:01 +0800 Subject: [PATCH 58/91] refactor: refactor workflows structure. (#2511) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. --- .github/.codecov.yml | 15 +- .github/ISSUE_TEMPLATE/bug-report.yml | 65 +++ .github/ISSUE_TEMPLATE/config.yml | 11 + .github/ISSUE_TEMPLATE/deployment.yml | 65 +++ .github/ISSUE_TEMPLATE/documentation.md | 20 + .github/ISSUE_TEMPLATE/feature-request.yml | 43 ++ .github/ISSUE_TEMPLATE/other.yml | 30 ++ .github/ISSUE_TEMPLATE/rfc.md | 26 + .github/code-language-detector.yml | 22 - .github/labels.yml | 43 -- .github/release-drafter.yml | 51 -- .github/standardizer.yml | 50 -- .github/sync-release.yml | 16 +- .github/sync.yml | 136 ----- .github/weekly-digest.yml | 21 - .github/workflows/auto-assign-issue.yml | 26 +- .github/workflows/auto-gh-pr.yml | 72 --- ...uto-invite.yml => auto-invite-comment.yml} | 36 +- .github/workflows/auto-tag.yml | 52 -- .github/workflows/bot-auto-cherry-pick.yml | 67 --- .github/workflows/bot-cherry-pick.yml | 68 --- .github/workflows/check-coverage.bak | 59 -- .github/workflows/cla-assistant.yml | 40 ++ .github/workflows/cla.yml | 62 --- .github/workflows/code-language-detector.yml | 27 - .github/workflows/codeql-analysis.yml | 103 ++-- .github/workflows/comment-check.yml | 51 ++ .github/workflows/create-branch-on-tag.bak | 77 --- .github/workflows/depsreview.yaml | 18 - .github/workflows/docker-buildx.bak | 502 ------------------ .github/workflows/e2e-test.bak | 159 ------ .github/workflows/go-build-test.yml | 135 +++++ .github/workflows/golangci-lint.bak | 58 -- .github/workflows/gosec.yml | 45 -- .github/workflows/issue-robot.yml | 31 -- .github/workflows/lock-issue.bak | 65 --- .github/workflows/milestone.yml | 74 --- .github/workflows/opencommit.yml | 55 -- .github/workflows/openimci.yml | 135 ----- .github/workflows/project-progress.yml | 39 -- ...ker-image.yml => publish-docker-image.yml} | 93 +--- .github/workflows/pull-request.bak | 130 ----- .github/workflows/release-drafter.yml | 55 -- .github/workflows/release.bak | 81 --- .github/workflows/remove-unused-labels.yml | 74 +++ .github/workflows/reopen-issue.yml | 78 +++ .github/workflows/sync-release.bak | 44 -- .github/workflows/sync.bak | 40 -- ...eetings.yml => user-first-interaction.yml} | 18 +- .golangci.yml | 19 +- 50 files changed, 727 insertions(+), 2575 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/deployment.yml create mode 100644 .github/ISSUE_TEMPLATE/documentation.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/ISSUE_TEMPLATE/other.yml create mode 100644 .github/ISSUE_TEMPLATE/rfc.md delete mode 100644 .github/code-language-detector.yml delete mode 100644 .github/labels.yml delete mode 100644 .github/release-drafter.yml delete mode 100644 .github/standardizer.yml delete mode 100644 .github/sync.yml delete mode 100644 .github/weekly-digest.yml delete mode 100644 .github/workflows/auto-gh-pr.yml rename .github/workflows/{auto-invite.yml => auto-invite-comment.yml} (74%) delete mode 100644 .github/workflows/auto-tag.yml delete mode 100644 .github/workflows/bot-auto-cherry-pick.yml delete mode 100644 .github/workflows/bot-cherry-pick.yml delete mode 100644 .github/workflows/check-coverage.bak create mode 100644 .github/workflows/cla-assistant.yml delete mode 100644 .github/workflows/cla.yml delete mode 100644 .github/workflows/code-language-detector.yml create mode 100644 .github/workflows/comment-check.yml delete mode 100644 .github/workflows/create-branch-on-tag.bak delete mode 100644 .github/workflows/depsreview.yaml delete mode 100644 .github/workflows/docker-buildx.bak delete mode 100644 .github/workflows/e2e-test.bak create mode 100644 .github/workflows/go-build-test.yml delete mode 100644 .github/workflows/golangci-lint.bak delete mode 100644 .github/workflows/gosec.yml delete mode 100644 .github/workflows/issue-robot.yml delete mode 100644 .github/workflows/lock-issue.bak delete mode 100644 .github/workflows/milestone.yml delete mode 100644 .github/workflows/opencommit.yml delete mode 100644 .github/workflows/openimci.yml delete mode 100644 .github/workflows/project-progress.yml rename .github/workflows/{build-docker-image.yml => publish-docker-image.yml} (63%) delete mode 100644 .github/workflows/pull-request.bak delete mode 100644 .github/workflows/release-drafter.yml delete mode 100644 .github/workflows/release.bak create mode 100644 .github/workflows/remove-unused-labels.yml create mode 100644 .github/workflows/reopen-issue.yml delete mode 100644 .github/workflows/sync-release.bak delete mode 100644 .github/workflows/sync.bak rename .github/workflows/{greetings.yml => user-first-interaction.yml} (65%) diff --git a/.github/.codecov.yml b/.github/.codecov.yml index fab584a316..c14ba471f5 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -1,17 +1,3 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - coverage: status: project: @@ -28,6 +14,7 @@ coverage: paths: - test/* # only include coverage in "test/" folder informational: true # Always pass check + # internal: # declare a new status context "internal" # paths: # - internal/* # only include coverage in "internal/" folder diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000000..2158804b76 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,65 @@ +name: Bug Report +title: "[BUG] " +labels: ["bug"] +description: "Create a detailed report to help us identify and resolve issues." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to fill out the bug report. Please provide as much information as possible to help us understand and replicate the bug." + + - type: input + id: openim-server-version + attributes: + label: OpenIM Server Version + description: "Please provide the version number of OpenIM Server you are using." + placeholder: "e.g., 3.8.0" + validations: + required: true + + - type: dropdown + id: operating-system + attributes: + label: Operating System and CPU Architecture + description: "Please select the operating system and describe the CPU architecture." + options: + - Linux (AMD) + - Linux (ARM) + - Windows (AMD) + - Windows (ARM) + - macOS (AMD) + - macOS (ARM) + validations: + required: true + + - type: dropdown + id: deployment-method + attributes: + label: Deployment Method + description: "Please specify how OpenIM Server was deployed." + options: + - Source Code Deployment + - Docker Deployment + validations: + required: true + + - type: textarea + id: bug-description-reproduction + attributes: + label: Bug Description and Steps to Reproduce + description: "Provide a detailed description of the bug and a step-by-step guide on how to reproduce it." + placeholder: "Describe the bug in detail here...\n\nSteps to reproduce the bug on the server:\n1. Start the server with specific configurations (mention any relevant config details).\n2. Make an API call to '...' endpoint with the following payload '...'.\n3. Observe the behavior and note any error messages or logs.\n4. Mention any additional setup relevant to the bug (e.g., database version, external service dependencies)." + validations: + required: true + + - type: markdown + attributes: + value: "If possible, please add screenshots to help explain your problem." + + - type: textarea + id: screenshots-link + attributes: + label: Screenshots Link + description: "If applicable, please provide any links to screenshots here." + placeholder: "Paste your screenshot URL here, e.g., http://imgur.com/example" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..deb8990831 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + # - name: "Bug Report" + # description: "Report a bug in the project" + # file: "bug-report.yml" + - name: 📢 Connect on slack + url: https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg + about: Support OpenIM-related requests or issues, get in touch with developers and help on slack + - name: 🌐 OpenIM Blog + url: https://www.openim.io/ + about: Open the OpenIM community blog diff --git a/.github/ISSUE_TEMPLATE/deployment.yml b/.github/ISSUE_TEMPLATE/deployment.yml new file mode 100644 index 0000000000..8df6243228 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/deployment.yml @@ -0,0 +1,65 @@ +name: Deployment issue +title: "[Deployment] " +labels: ["deployment"] +description: "Create a detailed report to help us identify and resolve deployment issues." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to fill out the deployment issue report. Please provide as much information as possible to help us understand and resolve the issue." + + - type: input + id: openim-server-version + attributes: + label: OpenIM Server Version + description: "Please provide the version number of OpenIM Server you are using." + placeholder: "e.g., 3.8.0" + validations: + required: true + + - type: dropdown + id: operating-system + attributes: + label: Operating System and CPU Architecture + description: "Please select the operating system and describe the CPU architecture." + options: + - Linux (AMD) + - Linux (ARM) + - Windows (AMD) + - Windows (ARM) + - macOS (AMD) + - macOS (ARM) + validations: + required: true + + - type: dropdown + id: deployment-method + attributes: + label: Deployment Method + description: "Please specify how OpenIM Server was deployed." + options: + - Source Code Deployment + - Docker Deployment + validations: + required: true + + - type: textarea + id: issue-description-reproduction + attributes: + label: Issue Description and Steps to Reproduce + description: "Provide a detailed description of the issue and a step-by-step guide on how to reproduce it." + placeholder: "Describe the issue in detail here...\n\nSteps to reproduce the issue on the server:\n1. Start the server with specific configurations (mention any relevant config details).\n2. Make an API call to '...' endpoint with the following payload '...'.\n3. Observe the behavior and note any error messages or logs.\n4. Mention any additional setup relevant to the bug (e.g., database version, external service dependencies)." + validations: + required: true + + - type: markdown + attributes: + value: "If possible, please add screenshots to help explain your problem." + + - type: textarea + id: screenshots-link + attributes: + label: Screenshots Link + description: "If applicable, please provide any links to screenshots here." + placeholder: "Paste your screenshot URL here, e.g., http://imgur.com/example" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..e6f751e4eb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,20 @@ +--- +name: Documentation Update +about: Propose updates to documentation, including README files and other docs. +title: "[DOC]: " # Prefix for the title to help identify documentation issues +labels: documentation # Labels to be automatically added +assignees: '' # Optionally, specify maintainers or teams to be auto-assigned + +--- + +## Documentation Updates +Describe the documentation that needs to be updated or corrected. Please specify the files and sections if possible. + +## Motivation +Explain why these updates are necessary. What is missing, misleading, or outdated? + +## Suggested Changes +Detail the changes that you propose. If you are suggesting large changes, include examples or mockups of what the updated documentation should look like. + +## Additional Information +Include any other information that might be relevant, such as links to discussions or related issues in the repository. diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000000..18a96a965c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,43 @@ +name: Feature Request +title: "[FEATURE REQUEST] " +labels: ["feature request","enhancement"] +description: "Propose a new feature or improvement that you believe will help enhance the project." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to propose a feature request. Please fill in as much detail as possible to help us understand why this feature is necessary and how it should work." + + - type: textarea + id: feature-reason + attributes: + label: Why this feature? + description: "Explain why this feature is needed. What problem does it solve? How does it benefit the project and its users?" + placeholder: "Describe the need for this feature..." + validations: + required: true + + - type: textarea + id: solution-proposal + attributes: + label: Suggested Solution + description: "Describe your proposed solution for this feature. How do you envision it working?" + placeholder: "Detail your solution here..." + validations: + required: true + + - type: markdown + attributes: + value: "Please provide any other relevant information or screenshots that could help illustrate your idea." + + - type: textarea + id: additional-info + attributes: + label: Additional Information + description: "Include any additional information, links, or screenshots that might be relevant to your feature request." + placeholder: "Add more context or links to relevant resources..." + + - type: markdown + attributes: + value: "Thank you for contributing to the project! We appreciate your input and will review your suggestion as soon as possible." diff --git a/.github/ISSUE_TEMPLATE/other.yml b/.github/ISSUE_TEMPLATE/other.yml new file mode 100644 index 0000000000..025440229a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other.yml @@ -0,0 +1,30 @@ +name: 🐧 Other +description: Use this for any other issues. Please do NOT create blank issues +title: "[Other]: " +labels: ["other"] +# assignees: [] + + +body: + - type: markdown + attributes: + value: "# Other issue" + - type: textarea + id: issuedescription + attributes: + label: What would you like to share? + description: Provide a clear and concise explanation of your issue. + validations: + required: true + - type: textarea + id: extrainfo + attributes: + label: Additional information + description: Is there anything else we should know about this issue? + validations: + required: false + - type: markdown + attributes: + value: | + You can also join our Discord community [here](https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg) + Feel free to check out other cool repositories of the openim Community [here](https://github.com/openimsdk) diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md new file mode 100644 index 0000000000..760d89e534 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -0,0 +1,26 @@ +--- +name: RFC - Feature Proposal +about: Submit a proposal for a significant feature to invite community discussion. +title: "[RFC]: " # Prefix for the title to help identify RFC proposals +labels: rfc, proposal # Labels to be automatically added +assignees: '' # Optionally, specify maintainers or teams to be auto-assigned + +--- + +## Proposal Overview +Briefly describe the content and objectives of your proposal. + +## Motivation +Why is this new feature necessary? What is the background of this problem? + +## Detailed Design +Describe the technical details of the proposal, including implementation steps, code snippets, or architecture diagrams. + +## Alternatives Considered +Have other alternatives been considered? Why is this approach preferred over others? + +## Impact +How will this proposal affect existing practices and community users? + +## Additional Information +Include any other relevant information such as related discussions, prior related work, etc. diff --git a/.github/code-language-detector.yml b/.github/code-language-detector.yml deleted file mode 100644 index 194c2474ad..0000000000 --- a/.github/code-language-detector.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright © 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/marketplace/actions/code-language-detector -directory: ./ -file_types: - - .go - - .yaml - - .yml -languages: - - Chinese \ No newline at end of file diff --git a/.github/labels.yml b/.github/labels.yml deleted file mode 100644 index b85a824b42..0000000000 --- a/.github/labels.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Refer to Kubernetes for size/* Settings -# https://github.com/Kubernetes/Kubernetes -XS: - name: size/XS - lines: 0 - color: 3CBF00 -S: - name: size/S - lines: 10 - color: 5D9801 -M: - name: size/M - lines: 30 - color: 7F7203 -L: - name: size/L - lines: 100 - color: A14C05 -XL: - name: size/XL - lines: 500 - color: C32607 -XXL: - name: size/XXL - lines: 1000 - color: E50009 - comment: | - # Whoa! Easy there, Partner! - This PR is too big. Please break it up into smaller PRs. \ No newline at end of file diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml deleted file mode 100644 index 55ee241d7b..0000000000 --- a/.github/release-drafter.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name-template: 'v$RESOLVED_VERSION 🌈' -tag-template: 'v$RESOLVED_VERSION' -categories: - - title: '🚀 Features' - labels: - - 'feature' - - 'enhancement' - - title: '🐛 Bug Fixes' - labels: - - 'kind/fix' - - 'kind/feature' - - 'enhancement' - - 'kind/documentation' - - 'good first issue' - - title: '🧰 Maintenance' - label: 'chore' -change-template: '- $TITLE @$AUTHOR (#$NUMBER)' -change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. -version-resolver: - major: - labels: - - 'major' - minor: - labels: - - 'minor' - patch: - labels: - - 'patch' - default: patch -template: | - ## Changes $PREVIOUS_TAG - - $CHANGES - - ## Contributors to this $REPOSITORY release - - $CONTRIBUTORS diff --git a/.github/standardizer.yml b/.github/standardizer.yml deleted file mode 100644 index fceb69df10..0000000000 --- a/.github/standardizer.yml +++ /dev/null @@ -1,50 +0,0 @@ -# https://github.com/marketplace/actions/conformity-checker-for-project -baseConfig: - searchDirectory: "./" - ignoreCase: false - -directoryNaming: - allowHyphens: true - allowUnderscores: false - mustBeLowercase: true - -fileNaming: - allowHyphens: true - allowUnderscores: true - mustBeLowercase: true - -ignoreFormats: - - "\\.log$" - - "\\.env$" - - "README\\.md$" - - "_test\\.go$" - - "\\.md$" - - _test\\.txt$ - - LICENSE - - Dockerfile - - CODEOWNERS - - Makefile - -ignoreDirectories: - - "vendor" - - ".git" - - "deployments" - - "node_modules" - - "logs" - - "CHANGELOG" - - "components" - - "_output" - - "tools/openim-web" - - "CHANGELOG" - - "examples/Test_directory" - - test/testdata - -fileTypeSpecificNaming: - ".yaml": - allowHyphens: true - allowUnderscores: false - mustBeLowercase: true - ".go": - allowHyphens: false - allowUnderscores: true - mustBeLowercase: true \ No newline at end of file diff --git a/.github/sync-release.yml b/.github/sync-release.yml index 3800b4c24b..18a80fda2c 100644 --- a/.github/sync-release.yml +++ b/.github/sync-release.yml @@ -1,18 +1,4 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OpenIMSDK/openim-docker: +openimsdk/openim-docker: - source: ./config dest: ./openim-server/release/config replace: true diff --git a/.github/sync.yml b/.github/sync.yml deleted file mode 100644 index ee667d415f..0000000000 --- a/.github/sync.yml +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -# Synchronization for the.github repository -OpenIMSDK/.github: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - -OpenIMSDK/community: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/ - dest: .github/workflows/ - -OpenIMSDK/openim-sdk-core: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/OpenIM-Docs: - - source: .github/workflows/ - dest: .github/workflows/ - exclude: | - e2e-test.yml - sync.yml - - source: scripts/githooks/ - dest: scripts/githooks/ - replace: true - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/OpenKF: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/openim-docker: - - source: ./config - dest: ./openim-server/main/config - replace: true - - source: ./docs - dest: ./openim-server/main/docs - replace: true - - source: ./scripts - dest: ./openim-server/main/scripts - replace: true - - source: ./scripts - dest: ./scripts - replace: true - - source: ./Makefile - dest: ./Makefile - replace: true - -group: - # first group：common to all warehouses - # TODO: add the required warehouse here - - repos: | - OpenIMSDK/OpenKF@main - OpenIMSDK/openim-miniprogram-demo@main - OpenIMSDK/docs - OpenIMSDK/chat - OpenIMSDK/community - OpenIMSDK/openim-charts - OpenIMSDK/openim-sdk-cpp@main - files: - - source: LICENSE - dest: LICENSE - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/workflows/project-progress.yml - dest: .github/workflows/project-progress.yml - replace: false - - source: .github/workflows/help-comment-issue.yml - dest: .github/workflows/help-comment-issue.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - - source: .github/workflows/cla.yml - dest: .github/workflows/cla.yml - replace: false - - source: .github/workflows/auto-assign-issue.yml - dest: .github/workflows/auto-assign-issue.yml - replace: false - - source: .github/workflows/release.yml - dest: .github/workflows/release.yml - replace: false - - source: ./scripts/githooks/ - dest: ./scripts/githooks/ - replace: true diff --git a/.github/weekly-digest.yml b/.github/weekly-digest.yml deleted file mode 100644 index fb3614ad81..0000000000 --- a/.github/weekly-digest.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/apps/weekly-digest/installations/new -publishDay: sun -canPublishIssues: true -canPublishPullRequests: true -canPublishContributors: true -canPublishStargazers: true -canPublishCommits: true \ No newline at end of file diff --git a/.github/workflows/auto-assign-issue.yml b/.github/workflows/auto-assign-issue.yml index d92fc968cb..320174d8c9 100644 --- a/.github/workflows/auto-assign-issue.yml +++ b/.github/workflows/auto-assign-issue.yml @@ -1,17 +1,3 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - name: Assign issue to comment author on: issue_comment: @@ -20,8 +6,7 @@ jobs: assign-issue: if: | contains(github.event.comment.body, '/assign') || contains(github.event.comment.body, '/accept') && - !contains(github.event.comment.user.login, 'openimbot') && - !contains(github.event.comment.user.login, 'kubbot') + !contains(github.event.comment.user.login, 'openim-robot') runs-on: ubuntu-latest permissions: issues: write @@ -33,11 +18,12 @@ jobs: run: | export LETASE_MILESTONES=$(curl 'https://api.github.com/repos/$OWNER/$PEPO/milestones' | jq -r 'last(.[]).title') gh issue edit ${{ github.event.issue.number }} --add-assignee "${{ github.event.comment.user.login }}" - gh issue edit ${{ github.event.issue.number }} --add-label "triage/accepted" - gh issue edit ${{ github.event.issue.number }} --milestone "$LETASE_MILESTONES" + gh issue edit ${{ github.event.issue.number }} --add-label "accepted" gh issue comment $ISSUE --body "@${{ github.event.comment.user.login }} Glad to see you accepted this issue🤲, this issue has been assigned to you. I set the milestones for this issue to [$LETASE_MILESTONES](https://github.com/$OWNER/$PEPO/milestones), We are looking forward to your PR!" + + # gh issue edit ${{ github.event.issue.number }} --milestone "$LETASE_MILESTONES" env: - GH_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.BOT_TOKEN }} ISSUE: ${{ github.event.issue.html_url }} OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} \ No newline at end of file + REPO: ${{ github.event.repository.name }} diff --git a/.github/workflows/auto-gh-pr.yml b/.github/workflows/auto-gh-pr.yml deleted file mode 100644 index 45454275ed..0000000000 --- a/.github/workflows/auto-gh-pr.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Auto PR to release - -on: - pull_request: - # types: - # - closed - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - -jobs: - sync-issue-to-pr: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Sync Issue to PR - if: github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' - run: | - PR_BODY="${{ github.event.pull_request.body }}" - - ISSUE_NUMBER=$(echo "$PR_BODY" | grep -oP 'Fixes #\K\d+') - if [[ -z "$ISSUE_NUMBER" ]]; then - echo "No Issue number found." - exit 1 - fi - - echo "Issue number found: $ISSUE_NUMBER" - - # Using GitHub CLI to get issue details - gh issue view "$ISSUE_NUMBER" --repo "${{ github.repository }}" --json labels,assignees,milestone,title > issue_data.json - - # Check if jq is installed - if ! command -v jq &> /dev/null; then - echo "Installing jq..." - sudo apt-get install -y jq - fi - - # Parse data with jq - LABELS=$(jq -r '.labels | map(.name) | join(",")' issue_data.json) - ASSIGNEES=$(jq -r '.assignees | map(.login) | join(",")' issue_data.json) - MILESTONE=$(jq -r '.milestone.title' issue_data.json) - - # Check if any of the fields are empty and set them to None - LABELS=${LABELS:-None} - ASSIGNEES=${ASSIGNEES:-None} - MILESTONE=${MILESTONE:-None} - - # Edit the PR with issue details, handling empty fields - gh pr edit "${{ github.event.pull_request.number }}" --repo "${{ github.repository }}" \ - ${LABELS:+--add-label "$LABELS"} \ - ${ASSIGNEES:+--add-assignee "$ASSIGNEES"} \ - ${MILESTONE:+--milestone "$MILESTONE"} - continue-on-error: true - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/auto-invite.yml b/.github/workflows/auto-invite-comment.yml similarity index 74% rename from .github/workflows/auto-invite.yml rename to .github/workflows/auto-invite-comment.yml index 350de30abc..76fbcdfd33 100644 --- a/.github/workflows/auto-invite.yml +++ b/.github/workflows/auto-invite-comment.yml @@ -1,32 +1,18 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Invite users to join our group +name: Invite users to join OpenIM Community. on: issue_comment: types: - created jobs: issue_comment: - name: Invite users to join our group + name: Invite users to join OpenIM Community if: ${{ github.event.comment.body == '/invite' || github.event.comment.body == '/close' || github.event.comment.body == '/comment' }} runs-on: ubuntu-latest permissions: issues: write steps: - - name: Invite user to join our group + - name: Invite user to join OpenIM Community uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.BOT_GITHUB_TOKEN }} @@ -43,11 +29,11 @@ jobs: + Read our [blog](https://doc.rentsoft.cn/). Our blog is a great place to stay up-to-date with Open-IM-Server projects and trends. On the blog, we share our latest developments, tech trends, and other interesting information. + Add [Wechat](https://github.com/OpenIMSDK/OpenIM-Docs/blob/main/docs/images/WechatIMG20.jpeg) and indicate that you are a user or developer of Open-IM-Server. We will process your request as soon as possible. - - name: Close Issue - uses: peter-evans/close-issue@v3 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - issue-number: ${{ github.event.issue.number }} - comment: 🤖 Auto-closing issue, if you still need help please reopen the issue or ask for help in the community above - labels: | - triage/accepted \ No newline at end of file + # - name: Close Issue + # uses: peter-evans/close-issue@v3 + # with: + # token: ${{ secrets.BOT_GITHUB_TOKEN }} + # issue-number: ${{ github.event.issue.number }} + # comment: 🤖 Auto-closing issue, if you still need help please reopen the issue or ask for help in the community above + # labels: | + # accepted \ No newline at end of file diff --git a/.github/workflows/auto-tag.yml b/.github/workflows/auto-tag.yml deleted file mode 100644 index 3decba7abc..0000000000 --- a/.github/workflows/auto-tag.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Create Tag - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - -jobs: - create_tag: - runs-on: ubuntu-latest - if: startsWith(github.event.comment.body, '/create tag') - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Validate version number and get comment - id: validate - run: | - COMMENT="${{ github.event.comment.body }}" - VERSION=$(echo $COMMENT | cut -d ' ' -f 3) - TAG_COMMENT=$(echo $COMMENT | cut -d '"' -f 2) - if [[ $VERSION =~ ^v([0-9]+\.){2}[0-9]+$ ]]; then - echo "version=$VERSION" >> $GITHUB_STATE - echo "tag_comment=$TAG_COMMENT" >> $GITHUB_STATE - else - echo "Invalid version number." - exit 1 - fi - - - name: Create a new tag - env: - GH_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - run: | - source $GITHUB_STATE - git tag -a $VERSION -m "$tag_comment" - git push origin $VERSION - echo "tag_created=$VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/bot-auto-cherry-pick.yml b/.github/workflows/bot-auto-cherry-pick.yml deleted file mode 100644 index cdd7241e2c..0000000000 --- a/.github/workflows/bot-auto-cherry-pick.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Rebot for Cherry Pick when PR is merged -on: - pull_request_target: - types: - - closed - -jobs: - comment: - runs-on: ubuntu-latest - steps: - - name: Comment cherry-pick command - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - script: | - const pr = context.payload.pull_request; - if (!pr.merged) { - console.log("PR is not merged. Skipping..."); - return; - } - if (!pr.milestone || !pr.milestone.title) { - console.log("Milestone is not set. Skipping..."); - return; - } - const milestone = pr.milestone.title; - const ref = `heads/release-${milestone}`; - let branchExists; - try { - await github.rest.git.getRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: ref - }); - branchExists = true; - } catch (error) { - if (error.status === 404) { - console.log(`Branch ${ref} does not exist. Skipping...`); - branchExists = false; - } else { - throw error; // Rethrow if it's another error - } - } - if (!branchExists) { - return; - } - const cherryPickCmd = `/cherry-pick release-${milestone}`; - console.log(`Adding comment: ${cherryPickCmd}`); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - body: cherryPickCmd - }); \ No newline at end of file diff --git a/.github/workflows/bot-cherry-pick.yml b/.github/workflows/bot-cherry-pick.yml deleted file mode 100644 index 71597189c6..0000000000 --- a/.github/workflows/bot-cherry-pick.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Robot for Cherry Pick On Comment - -on: - issue_comment: - types: [created] - -jobs: - cherry-pick: - name: Cherry Pick - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/cherry-pick') - runs-on: ubuntu-latest - - steps: - - name: Checkout the latest code - uses: actions/checkout@v4 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - fetch-depth: 0 # To ensure all history is available for cherry-picking - - - name: Automatic Cherry Pick - uses: vendoo/gha-cherry-pick@v1 - with: - # Assuming the cherry-pick commit SHA is passed in the comment like '/cherry-pick sha' - commit-sha: ${{ github.event.comment.body }} - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - - name: Create a new branch for PR - run: | - PR_BRANCH="cherry-pick-${GITHUB_SHA}-to-${{ github.base_ref }}" - git checkout -b $PR_BRANCH - git push origin $PR_BRANCH - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - - name: Create Pull Request - uses: actions/github-script@v5 - with: - script: | - const prTitle = "Cherry-pick to ${{ github.base_ref }}" - const prBody = "Automated cherry-pick of ${{ github.event.comment.body }}\n\n/cc @kubbot" - const base = "${{ github.base_ref }}" - const head = "cherry-pick-${{ github.sha }}-to-${{ github.base_ref }}" - const createPr = await github.rest.pulls.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: prTitle, - body: prBody, - head: head, - base: base, - maintainer_can_modify: true, // Allows maintainers to edit the PR - }) - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/check-coverage.bak b/.github/workflows/check-coverage.bak deleted file mode 100644 index 09d43d7cdf..0000000000 --- a/.github/workflows/check-coverage.bak +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Check Coverage - -on: - workflow_dispatch: - push: - branches: [ "main" ] - paths-ignore: - - "docs/**" - - "**/*.md" - - "**/*.yaml" - - "CONTRIBUTORS" - - "CHANGELOG/**" - pull_request: - branches: [ "*" ] - paths-ignore: - - "docs/**" - - "**/*.md" - - "**/*.yaml" - - "CONTRIBUTORS" - - "CHANGELOG/**" -env: - # Common versions - GO_VERSION: "1.20" - -jobs: - coverage: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Golang with cache - uses: magnetikonline/action-golang-cache@v4 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - run: sudo apt update && sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev - - - name: Run Cover - run: make cover - continue-on-error: true - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v4 diff --git a/.github/workflows/cla-assistant.yml b/.github/workflows/cla-assistant.yml new file mode 100644 index 0000000000..71bdb67992 --- /dev/null +++ b/.github/workflows/cla-assistant.yml @@ -0,0 +1,40 @@ +name: CLA Assistant +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened,closed,synchronize] + +# explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings +permissions: + actions: write + contents: write # this can be 'read' if the signatures are in remote repository + pull-requests: write + statuses: write + +jobs: + CLA-Assistant: + runs-on: ubuntu-latest + steps: + - name: "CLA Assistant" + if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' + uses: contributor-assistant/github-action@v2.4.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.BOT_TOKEN }} + with: + path-to-signatures: 'signatures/cla.json' + path-to-document: 'https://github.com/OpenIM-Robot/cla/blob/main/README.md' # e.g. a CLA or a DCO document + branch: 'main' + allowlist: 'bot*,*bot,OpenIM-Robot' + + # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken + remote-organization-name: OpenIM-Robot + remote-repository-name: cla + create-file-commit-message: 'Creating file for storing CLA Signatures' + # signed-commit-message: '$contributorName has signed the CLA in $owner/$repo#$pullRequestNo' + custom-notsigned-prcomment: '💕 Thank you for your contribution and please kindly read and sign our [CLA Docs](https://github.com/OpenIM-Robot/cla/blob/main/README.md)' + custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' + custom-allsigned-prcomment: '🤖 All Contributors have signed the [CLA](https://github.com/OpenIM-Robot/cla/blob/main/README.md).
The signed information is recorded [🤖here](https://github.com/openim-sigs/cla/tree/main/signatures/cla.json)' + #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) + #use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml deleted file mode 100644 index ae27c8a0a5..0000000000 --- a/.github/workflows/cla.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM CLA Assistant -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened,closed,synchronize] - -# explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings -permissions: - actions: write - contents: write - pull-requests: write - statuses: write - -env: - # Define Open-IM-Server variables here - OPEN_IM_SERVER_REMOTE_ORGANIZATION: openim-sigs - REMOTE_REPOSITORY: cla - OPEN_IM_SERVER_CLA_DOCUMENT: https://github.com/openim-sigs/cla/blob/main/README.md - OPEN_IM_SERVER_SIGNATURES_PATH: signatures/${{ github.event.repository.name }}/cla.json - - OPEN_IM_SERVER_ALLOWLIST: kubbot,openimbot,bot*,dependabot,sweep-ai,*bot,bot-*,bot/*,bot-/*,bot,*[bot] - -jobs: - CLAAssistant: - runs-on: ubuntu-latest - steps: - - name: "CLA Assistant" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: contributor-assistant/github-action@v2.3.1 - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} - with: - path-to-signatures: ${{ env.OPEN_IM_SERVER_SIGNATURES_PATH }} - path-to-document: ${{ env.OPEN_IM_SERVER_CLA_DOCUMENT }} - branch: 'main' - allowlist: ${{ env.OPEN_IM_SERVER_ALLOWLIST }} - - remote-organization-name: ${{ env.OPEN_IM_SERVER_REMOTE_ORGANIZATION }} - remote-repository-name: ${{ env.REMOTE_REPOSITORY }} - - create-file-commit-message: '📚 Docs: Creating file for storing ${{ github.event.repository.name }} CLA Signatures' - custom-notsigned-prcomment: '💕 Thank you for your contribution and please kindly read and sign our [🎯https://github.com/openim-sigs/cla/blob/main/README.md](https://github.com/openim-sigs/cla/blob/main/README.md).
If you wish to sign the CRA, **Please copy and comment on the following sentence:**' - custom-pr-sign-comment: 'I have read the CLA Document and I hereby sign the CLA' - custom-allsigned-prcomment: '🤖 All Contributors have signed the [${{ github.event.repository.name }} CLA](https://github.com/openim-sigs/cla/blob/main/README.md).
The signed information is recorded [🤖here](https://github.com/openim-sigs/cla/tree/main/signatures/${{ github.event.repository.name }}/cla.json)' - # lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) - # use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/code-language-detector.yml b/.github/workflows/code-language-detector.yml deleted file mode 100644 index 80ec947338..0000000000 --- a/.github/workflows/code-language-detector.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright © 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Language Check Workflow Test - -on: [pull_request] - -jobs: - comment-language-detector: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Code Language Detector - uses: kubecub/comment-lang-detector@v1.0.0 \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 29f9382ccc..fd871e2b58 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,76 +1,67 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. -name: "OpenIM Code Scanning - Action" +name: "CodeQL" on: push: - branches: [main] + branches: [ main ] pull_request: - branches: [main] + # The branches below must be a subset of the branches above + branches: [ main ] schedule: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - # * * * * * - - cron: '30 1 * * 0' + - cron: '18 19 * * 6' jobs: - CodeQL-Build: - # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + analyze: + name: Analyze runs-on: ubuntu-latest - permissions: - # required for all workflows - security-events: write - - # only required for workflows in private repositories - actions: write - contents: write + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - # Override language selection by uncommenting this and choosing your languages - with: - languages: go + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below). - - name: Autobuild - uses: github/codeql-action/autobuild@v3 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏️ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - # - run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 \ No newline at end of file + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 \ No newline at end of file diff --git a/.github/workflows/comment-check.yml b/.github/workflows/comment-check.yml new file mode 100644 index 0000000000..e994b52596 --- /dev/null +++ b/.github/workflows/comment-check.yml @@ -0,0 +1,51 @@ +name: Non-English Comments Check + +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + non-english-comments-check: + runs-on: ubuntu-latest + + env: + # need ignore Dirs + EXCLUDE_DIRS: ".git docs tests scripts assets node_modules build" + # need ignore Files + EXCLUDE_FILES: "*.md *.txt *.html *.css *.min.js *.mdx" + + steps: + - uses: actions/checkout@v4 + + - name: Search for Non-English comments + run: | + set -e + # Define the regex pattern to match Chinese characters + pattern='[\p{Han}]' + + # Process the directories to be excluded + exclude_dirs="" + for dir in $EXCLUDE_DIRS; do + exclude_dirs="$exclude_dirs --exclude-dir=$dir" + done + + # Process the file types to be excluded + exclude_files="" + for file in $EXCLUDE_FILES; do + exclude_files="$exclude_files --exclude=$file" + done + + # Use grep to find all comments containing Non-English characters and save to file + grep -Pnr "$pattern" . $exclude_dirs $exclude_files > non_english_comments.txt || true + + - name: Output non-English comments are found + run: | + if [ -s non_english_comments.txt ]; then + echo "Non-English comments found in the following locations:" + cat non_english_comments.txt + exit 1 # terminate the workflow + else + echo "No Non_English comments found." + fi diff --git a/.github/workflows/create-branch-on-tag.bak b/.github/workflows/create-branch-on-tag.bak deleted file mode 100644 index fbacd261b1..0000000000 --- a/.github/workflows/create-branch-on-tag.bak +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Create Branch on Tag - -on: - push: - tags: - - 'v*.*.0' - -permissions: - contents: write - actions: write - -jobs: - create-branch: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Git - run: | - git config --global user.name 'kubbot' - git config --global user.email '3293172751yxy@gmail.com' - - - name: Install git-chglog - run: make install.git-chglog - - - name: Create Branch and Push - env: - TAG_NAME: ${{ github.ref_name }} - run: | - IFS='.' read -ra VERSION_PARTS <<< "$TAG_NAME" - if [[ "${VERSION_PARTS[2]}" = "0" ]]; then - BRANCH_NAME="release-v${VERSION_PARTS[0]}.${VERSION_PARTS[1]}" - echo "Creating branch $BRANCH_NAME" - git checkout -b "$BRANCH_NAME" - git push origin "$BRANCH_NAME" - else - echo "Not a release tag. Skipping branch creation." - fi - continue-on-error: true - - - name: Create and Commit CHANGELOG - if: endsWith(github.ref_name, '.0') - run: | - git fetch --all - TAG_NAME=${GITHUB_REF#refs/tags/} - IFS='.' read -ra VERSION_PARTS <<< "$TAG_NAME" - git checkout main - cd CHANGELOG - git-chglog --tag-filter-pattern "v${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.*" -o "CHANGELOG-${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.md" - git add "CHANGELOG-${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.md" - git commit -m "Update CHANGELOG for $TAG_NAME" || echo "No changes to commit." - continue-on-error: true - - - name: Push CHANGELOG to Main - if: steps.create-and-commit-changelog.outputs.changes == 'true' - uses: ad-m/github-push-action@v0.8.0 - with: - github_token: ${{ secrets.BOT_GITHUB_TOKEN }} - branch: main - continue-on-error: true diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml deleted file mode 100644 index aff7e3d9e0..0000000000 --- a/.github/workflows/depsreview.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright © 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -name: OpenIM Dependency Review -on: [pull_request] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v4 - - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 \ No newline at end of file diff --git a/.github/workflows/docker-buildx.bak b/.github/workflows/docker-buildx.bak deleted file mode 100644 index 7e7b8229ca..0000000000 --- a/.github/workflows/docker-buildx.bak +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Docker Buildx Images CI - -on: - schedule: - - cron: '30 1 * * *' - push: - branches: - - release-* - tags: - - v* - workflow_dispatch: - -jobs: - build-ghcr: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - install: true - - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Log in to AliYun Docker Hub - uses: docker/login-action@v3 - with: - registry: registry.cn-hangzhou.aliyuncs.com - username: ${{ secrets.ALIREGISTRY_USERNAME }} - password: ${{ secrets.ALIREGISTRY_TOKEN }} - -################################################ -# build/ -# └── docker -# ├── openim-api -# │ └── Dockerfile -# ├── openim-cmdutils -# │ └── Dockerfile -# ├── openim-crontask -# │ └── Dockerfile -# ├── openim-msggateway -# │ └── Dockerfile -# ├── openim-msgtransfer -# │ └── Dockerfile -# ├── openim-push -# │ └── Dockerfile -# ├── openim-rpc-auth -# │ └── Dockerfile -# ├── openim-rpc-conversation -# │ └── Dockerfile -# ├── openim-rpc-friend -# │ └── Dockerfile -# ├── openim-rpc-group -# │ └── Dockerfile -# ├── openim-rpc-msg -# │ └── Dockerfile -# ├── openim-rpc-third -# │ └── Dockerfile -# └── openim-rpc-user -# └── Dockerfile -############################################# - - - name: Extract metadata (tags, labels) for Docker openim-api - id: meta1 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-api - openim/openim-api - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-api - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-api - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-api/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta1.outputs.tags }} - labels: ${{ steps.meta1.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-cmdutils - id: meta2 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-cmdutils - openim/openim-cmdutils - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-cmdutils - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-cmdutils - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-cmdutils/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta2.outputs.tags }} - labels: ${{ steps.meta2.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-crontask - id: meta3 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-crontask - openim/openim-crontask - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-crontask - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-crontask - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-crontask/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta3.outputs.tags }} - labels: ${{ steps.meta3.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-msggateway - id: meta4 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-msggateway - openim/openim-msggateway - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-msggateway - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-msggateway - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-msggateway/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta4.outputs.tags }} - labels: ${{ steps.meta4.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-msgtransfer - id: meta5 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-msgtransfer - openim/openim-msgtransfer - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-msgtransfer - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-msgtransfer - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-msgtransfer/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta5.outputs.tags }} - labels: ${{ steps.meta5.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-push - id: meta6 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-push - openim/openim-push - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-push - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-push - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-push/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta6.outputs.tags }} - labels: ${{ steps.meta6.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-auth - id: meta7 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-auth - openim/openim-rpc-auth - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-auth - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-auth - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-auth/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta7.outputs.tags }} - labels: ${{ steps.meta7.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-conversation - id: meta8 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-conversation - openim/openim-rpc-conversation - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-conversation - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-conversation - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-conversation/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta8.outputs.tags }} - labels: ${{ steps.meta8.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-friend - id: meta9 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-friend - openim/openim-rpc-friend - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-friend - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-friend - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-friend/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta9.outputs.tags }} - labels: ${{ steps.meta9.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-group - id: meta10 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-group - openim/openim-rpc-group - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-group - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-group - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-group/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta10.outputs.tags }} - labels: ${{ steps.meta10.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-msg - id: meta11 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-msg - openim/openim-rpc-msg - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-msg - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-msg - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-msg/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta11.outputs.tags }} - labels: ${{ steps.meta11.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-third - id: meta12 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-third - openim/openim-rpc-third - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-third - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-third - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-third/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta12.outputs.tags }} - labels: ${{ steps.meta12.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-user - id: meta13 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-user - openim/openim-rpc-user - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-user - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-user - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-user/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta13.outputs.tags }} - labels: ${{ steps.meta13.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/e2e-test.bak b/.github/workflows/e2e-test.bak deleted file mode 100644 index 6231697c25..0000000000 --- a/.github/workflows/e2e-test.bak +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM E2E And API Test - -on: - workflow_dispatch: - pull_request: - push: - schedule: - # run e2e test every 4 hours - - cron: 0 */4 * * * - -env: - CALLBACK_ENABLE: true - -jobs: - build: - name: Test - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - - name: Set up Go 1.21 - uses: actions/setup-go@v5 - with: - go-version: 1.21 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Create e2e test - run: | - echo "...test e2e" - - execute-linux-systemd-scripts: - name: Execute OpenIM script on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - environment: - name: openim - strategy: - matrix: - go_version: ["1.20"] - os: ["ubuntu-latest"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go ${{ matrix.go_version }} - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go_version }} - id: go - - - name: Install Task - uses: arduino/setup-task@v1 - with: - version: '3.x' # If available, use the latest major version that's compatible - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Operations - run: | - sudo docker compose up -d - sudo bash bootstrap.sh - sudo mage - sudo sleep 20 - - - name: Module Operations - run: | - echo "===========> Verifying go-gitlint is installed" - if [ ! -f ./_output/tools/go-gitlint ]; then - export GOBIN=$(pwd)/_output/tools - echo "===========> Installing The default installation path is /home/ubuntu/DF/open-im-server/_output/tools/go-gitlint" - sudo go install github.com/marmotedu/go-gitlint/cmd/go-gitlint@latest - echo "===========> go-gitlint is installed in /home/ubuntu/DF/open-im-server/_output/tools/go-gitlint" - fi - - - name: Build, Start(make build && make start) - run: | - sudo ./scripts/install/install.sh -i - - - name: Exec OpenIM System Status Chack - run: | - sudo ./scripts/install/install.sh -s - -# - name: Exec OpenIM API test (make test-api) - - name: Exec OpenIM test (make test) - run: | - mkdir -p ./tmp - touch ./tmp/test.md - echo "# OpenIM Test" >> ./tmp/test.md - echo "## OpenIM API Test" >> ./tmp/test.md - echo "

Command Output for OpenIM API Test" >> ./tmp/test.md - echo "
" >> ./tmp/test.md
-        echo "===========> Run api test"
-        ./scripts/install/test.sh
-        echo "===========> Run api test" >> ./tmp/test.md
-        ./scripts/install/test.sh >> ./tmp/test.md
-        echo "
" >> ./tmp/test.md - echo "
" >> ./tmp/test.md - - echo "===========> Run api test" - ./scripts/install/test.sh - - # - name: Exec OpenIM E2E Test (make test-e2e) - # run: | - # echo "" >> ./tmp/test.md - # echo "## OpenIM E2E Test" >> ./tmp/test.md - # echo "
Command Output for OpenIM E2E Test" >> ./tmp/test.md - # echo "
" >> ./tmp/test.md
- #       sudo make test-e2e | tee -a ./tmp/test.md
- #       echo "
" >> ./tmp/test.md - # echo "
" >> ./tmp/test.md - - # sudo make test-e2e - - - name: Comment PR with file - uses: thollander/actions-comment-pull-request@v2 - with: - filePath: ./tmp/test.md - comment_tag: nrt_file - reactions: eyes, rocket - mode: recreate - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - continue-on-error: true - - - name: Check outputs - run: | - echo "id : ${{ steps.nrt_message.outputs.id }}" - echo "body : ${{ steps.nrt_message.outputs.body }}" - echo "html_url : ${{ steps.nrt_message.outputs.html_url }}" - - - name: Exec OpenIM System uninstall - run: | - sudo ./scripts/install/install.sh -u - - - name: gobenchdata publish - uses: bobheadxi/gobenchdata@v1 - with: - PRUNE_COUNT: 30 - GO_TEST_FLAGS: -cpu 1,2 - PUBLISH: true - PUBLISH_BRANCH: gh-pages - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - continue-on-error: true diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml new file mode 100644 index 0000000000..14546f4f45 --- /dev/null +++ b/.github/workflows/go-build-test.yml @@ -0,0 +1,135 @@ +name: Go Build Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + go-build: + name: Test with go ${{ matrix.go_version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + contents: write + pull-requests: write + strategy: + matrix: + os: [ubuntu-latest] + go_version: ["1.21.x", "1.22.x"] + + steps: + - name: Checkout Server repository + uses: actions/checkout@v4 + + - name: Set up Go ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Get Server dependencies + run: | + go install github.com/magefile/mage@latest + go mod tidy + go mod download + + - name: Set up infra services + uses: hoverkraft-tech/compose-action@v2.0.1 + # Uncomment and set the correct path to your docker-compose file + with: + compose-file: "./docker-compose.yml" + + # run: | + # sudo docker compose up -d + # sudo sleep 30 # Increased sleep time for better stability + # timeout-minutes: 60 # Increased timeout for Docker setup + + + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" + + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml + + - name: Build and test Server Services + run: | + mage build + mage start + mage check + + - name: Checkout Chat repository + uses: actions/checkout@v4 + with: + repository: "openimsdk/chat" + path: "chat-repo" + + - name: Get Chat dependencies + run: | + cd ${{ github.workspace }}/chat-repo + go mod tidy + go mod download + go install github.com/magefile/mage@latest + + - name: Build and test Chat Services + run: | + cd ${{ github.workspace }}/chat-repo + mage build + mage start + mage check + + dockerfile-test: + name: Build and Test Dockerfile + runs-on: ubuntu-latest + strategy: + matrix: + go_version: ["1.21"] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Go ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Get dependencies + run: | + go mod tidy + go mod download + go install github.com/magefile/mage@latest + + - name: Build Docker Image + run: | + IMAGE_NAME="${{ github.event.repository.name }}-test" + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker build -t $IMAGE_NAME . + + - name: Run Docker Container + run: | + IMAGE_NAME="${{ github.event.repository.name }}-test" + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker run --name $CONTAINER_NAME -d $IMAGE_NAME + docker ps -a + + - name: Test Docker Container Logs + run: | + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker logs $CONTAINER_NAME + + # - name: Cleanup Docker Container + # run: | + # CONTAINER_NAME="${{ github.event.repository.name }}-container" + # IMAGE_NAME="${{ github.event.repository.name }}-test" + # docker stop $CONTAINER_NAME + # docker rm $CONTAINER_NAME + # docker rmi $IMAGE_NAME diff --git a/.github/workflows/golangci-lint.bak b/.github/workflows/golangci-lint.bak deleted file mode 100644 index 64bd498c54..0000000000 --- a/.github/workflows/golangci-lint.bak +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -name: OpenIM golangci-lint -on: - push: - branches: [main] - pull_request: -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: false - - name: OpenIM Scripts Verification(make verify) - run: | - cd scripts - for script in verify-*; do - if [ -x "$script" ]; then - ./"$script" - fi - done - - name: golangci-lint - uses: golangci/golangci-lint-action@v4.0.0 - with: - # Require: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.54 - - # Optional: working directory, useful for monorepos - # working-directory: server - - # Optional: golangci-lint command line arguments. - # - # Note: by default the `.golangci.yml` file should be at the root of the repository. - # The location of the configuration file can be changed by using `--config=` - # args: --timeout=30m --config=/scripts/.golangci.yml --issues-exit-code=0 - - # Optional: show only new issues if it's a pull request. The default value is `false`. - only-new-issues: true - - # Optional:The mode to install golangci-lint. It can be 'binary' or 'goinstall'. - # install-mode: "goinstall" diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml deleted file mode 100644 index b99330c05c..0000000000 --- a/.github/workflows/gosec.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Run Gosec - -# gosec is a source code security audit tool for the Go language. It performs a static -# analysis of the Go code, looking for potential security problems. The main functions of gosec are: -# 1. Find common security vulnerabilities, such as SQL injection, command injection, and cross-site scripting (XSS). -# 2. Audit codes according to common security standards and find non-standard codes. -# 3. Assist the Go language engineer to write safe and reliable code. -# https://github.com/securego/gosec/ -on: - push: - branches: "*" - pull_request: - branches: "*" - paths-ignore: - - '*.md' - - '*.yml' - - '.github' - -jobs: - golang-security-action: - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: ./... - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/issue-robot.yml b/.github/workflows/issue-robot.yml deleted file mode 100644 index 2a956ed19c..0000000000 --- a/.github/workflows/issue-robot.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Issue Aotu Translator -on: - issue_comment: - types: [created] - issues: - types: [opened] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: usthe/issues-translate-action@v2.7 - with: - # it is not necessary to decide whether you need to modify the issue header content - IS_MODIFY_TITLE: true - BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - # Required, input your bot github token \ No newline at end of file diff --git a/.github/workflows/lock-issue.bak b/.github/workflows/lock-issue.bak deleted file mode 100644 index edf2809653..0000000000 --- a/.github/workflows/lock-issue.bak +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: 'Lock Threads' - -on: - schedule: - - cron: '0 * * * *' - workflow_dispatch: - -permissions: - issues: write - pull-requests: write - -concurrency: - group: lock - -jobs: - action: - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v5 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - issue-inactive-days: '365' - exclude-issue-created-before: '' - exclude-issue-created-after: '' - exclude-issue-created-between: '' - exclude-issue-closed-before: '' - exclude-issue-closed-after: '' - exclude-issue-closed-between: '' - include-any-issue-labels: '' - include-all-issue-labels: '' - exclude-any-issue-labels: '' - add-issue-labels: '' - remove-issue-labels: '' - issue-comment: '' - issue-lock-reason: 'resolved' - pr-inactive-days: '365' - exclude-pr-created-before: '' - exclude-pr-created-after: '' - exclude-pr-created-between: '' - exclude-pr-closed-before: '' - exclude-pr-closed-after: '' - exclude-pr-closed-between: '' - include-any-pr-labels: '' - include-all-pr-labels: '' - exclude-any-pr-labels: '' - add-pr-labels: '' - remove-pr-labels: '' - pr-comment: '' - pr-lock-reason: 'resolved' - process-only: '' - log-output: false \ No newline at end of file diff --git a/.github/workflows/milestone.yml b/.github/workflows/milestone.yml deleted file mode 100644 index c74e7074a7..0000000000 --- a/.github/workflows/milestone.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# shamelessly copied from https://github.com/sigstore/cosign/blob/main/.github/workflows/milestone.yaml - -name: milestone - -on: - pull_request_target: - types: [closed] - branches: - - main - -jobs: - milestone: - runs-on: ubuntu-latest - - permissions: - actions: none - checks: none - contents: read - deployments: none - issues: write - packages: none - pull-requests: write - repository-projects: none - security-events: none - statuses: none - - steps: - - uses: actions/github-script@v7 # v6 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - script: | - if (!context.payload.pull_request.merged) { - console.log('PR was not merged, skipping.'); - return; - } - - if (!!context.payload.pull_request.milestone) { - console.log('PR has existing milestone, skipping.'); - return; - } - - milestones = await github.rest.issues.listMilestones({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - sort: 'title', - direction: 'desc' - }) - - if (milestones.data.length === 0) { - console.log('There are no milestones, skipping.'); - return; - } - - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - milestone: milestones.data[0].number - }); diff --git a/.github/workflows/opencommit.yml b/.github/workflows/opencommit.yml deleted file mode 100644 index d483ef1f65..0000000000 --- a/.github/workflows/opencommit.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM OpenCommit Action - -on: - push: - # this list of branches is often enough, - # but you may still ignore other public branches - branches-ignore: [main master dev development release] - -jobs: - opencommit: - timeout-minutes: 10 - name: OpenCommit - runs-on: ubuntu-latest - permissions: write-all - steps: - - name: Setup Node.js Environment - uses: actions/setup-node@v4 - with: - node-version: '16' - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: di-sukharev/opencommit@github-action-v1.0.4 - with: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - env: - # set openAI api key in repo actions secrets, - # for openAI keys go to: https://platform.openai.com/account/api-keys - # for repo secret go to: /settings/secrets/actions - OCO_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - # customization - OCO_OPENAI_MAX_TOKENS: 500 - OCO_OPENAI_BASE_PATH: '' - OCO_DESCRIPTION: false - OCO_EMOJI: false - OCO_MODEL: gpt-3.5-turbo-16k - OCO_LANGUAGE: en - OCO_PROMPT_MODULE: conventional-commit - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml deleted file mode 100644 index 8f3630dd08..0000000000 --- a/.github/workflows/openimci.yml +++ /dev/null @@ -1,135 +0,0 @@ - -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: OpenIM CI Auto Build - -on: - push: - branches: - - main - - release-* - paths-ignore: - - "docs/**" - - "README.md" - - "README_zh-CN.md" - - "**.md" - - "docs/**" - - "CONTRIBUTING.md" - pull_request: - branches: - - main - - release-* - paths-ignore: - - "README.md" - - "README_zh-CN.md" - - "CONTRIBUTING/**" - - "**.md" - - "docs/**" - workflow_dispatch: - -jobs: - - build-linux: - name: Execute OpenIM Script On Linux - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - environment: - name: openim - strategy: - matrix: - arch: [arm64, armv7, amd64] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - - name: Set up Docker for Linux - run: | - sudo docker compose up -d - sudo sleep 30 # Increased sleep time for better stability - timeout-minutes: 20 # Increased timeout for Docker setup - - - - name: init - run: sudo bash bootstrap.sh - timeout-minutes: 20 - - # - name: Get Internal IP Address - # id: get-ip - # run: | - # IP=$(hostname -I | awk '{print $1}') - # echo "The IP Address is: $IP" - # echo "::set-output name=ip::$IP" - - # - name: Update .env - # run: | - # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml - # cat config/minio.yml - - - name: Build, Start, Check Services and Print Logs for Linux - run: | - sudo mage - sudo mage start - sudo mage check - - - - name: Restart Services and Print Logs - run: | - sudo mage stop - sudo mage start - sudo mage check - - - name: Checkout chat repository - uses: actions/checkout@v4 - with: - repository: 'openimsdk/chat' - path: 'chat-repo' - - - name: Build and Start Chat Services - run: | - cd ${{ github.workspace }}/chat-repo - sudo mage - sudo mage start - sudo mage check - - # - name: Checkout e2e repository - # uses: actions/checkout@v4 - # with: - # repository: "openimsdk/test-e2e" - # path: e2e-repo - - # - name: Set up Python 3.9 - # uses: actions/setup-python@v4 - # with: - # python-version: '3.9' - - # - name: Install dependencies - # run: | - # sudo apt-get update - # sudo apt-get install -y xvfb libxi6 libgconf-2-4 - # cd ${{ github.workspace }}/e2e-repo - # pip install -r requirements.txt - - # - name: Run tests - # run: | - # cd ${{ github.workspace }}/e2e-repo - # xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script - - \ No newline at end of file diff --git a/.github/workflows/project-progress.yml b/.github/workflows/project-progress.yml deleted file mode 100644 index 87a4c13810..0000000000 --- a/.github/workflows/project-progress.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GitHub recommends pinning actions to a commit SHA. -# To get a newer version, you will need to update the SHA. -# You can also reference a tag or branch, but the action may change without warning. - -name: Move assigned card -on: - issues: - types: - - assigned - pull_request: - types: - - assigned - branches-ignore: - - 'asf-auto-updates' - - 'ignore' - -jobs: - move-assigned-card: - runs-on: ubuntu-latest - steps: - - uses: alex-page/github-project-automation-plus@v0.9.0 - with: - project: openim-powerful - column: In Progress - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/publish-docker-image.yml similarity index 63% rename from .github/workflows/build-docker-image.yml rename to .github/workflows/publish-docker-image.yml index d0b9dddbc4..7d7d23f173 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -1,63 +1,34 @@ -# Copyright © 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Publish Docker image +name: Publish Docker image to registries on: push: branches: - - main - release-* - paths-ignore: - - "docs/**" - - "README.md" - - "README_zh-CN.md" - - "**.md" - - "docs/**" - - "CONTRIBUTING.md" - tags: - - v* - pull_request: - types: [closed] - branches: - - main - - release-* - paths-ignore: - - "docs/**" - - "README.md" - - "README_zh-CN.md" - - "**.md" - - "docs/**" - - "CONTRIBUTING.md" + + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Tag version to be used for Docker image" + required: true + default: "v3.8.0" -env: - # Common versions - GO_VERSION: "1.20" +# env: +# GO_VERSION: "1.21" jobs: - build-dockerhub: + publish-docker-images: runs-on: ubuntu-latest - if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.merged == false) }} steps: - - name: Checkout main repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: path: main-repo - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -90,7 +61,7 @@ jobs: run: | cd ${{ github.workspace }}/compose-repo docker compose up -d - sleep 30 + sleep 60 - name: Check openim-server health run: | @@ -121,31 +92,8 @@ jobs: exit 0 fi - # - name: Checkout e2e - # if: success() - # uses: actions/checkout@v4 - # with: - # repository: "openimsdk/test-e2e" - # path: e2e-repo - - # - name: Set up Python 3.9 - # uses: actions/setup-python@v4 - # with: - # python-version: '3.9' - - # - name: Install dependencies - # run: | - # sudo apt-get update - # sudo apt-get install -y xvfb libxi6 libgconf-2-4 - # cd ${{ github.workspace }}/e2e-repo - # pip install -r requirements.txt - - # - name: Run tests - # run: | - # cd ${{ github.workspace }}/e2e-repo - # xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script - - - name: Extract metadata (tags, labels) for Docker + + - name: Extract metadata for Docker # (tags, labels) if: success() id: meta uses: docker/metadata-action@v5.5.1 @@ -154,6 +102,7 @@ jobs: openim/openim-server ghcr.io/openimsdk/openim-server registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server + # generate Docker tags based on the following events/attributes tags: | type=ref,event=tag @@ -186,13 +135,11 @@ jobs: username: ${{ secrets.ALIREGISTRY_USERNAME }} password: ${{ secrets.ALIREGISTRY_TOKEN }} - - name: Build and push Docker image + - name: Build and push Docker images uses: docker/build-push-action@v5 with: context: ./main-repo push: true - # linux/ppc64le,linux/s390x platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - diff --git a/.github/workflows/pull-request.bak b/.github/workflows/pull-request.bak deleted file mode 100644 index f7c5900ce7..0000000000 --- a/.github/workflows/pull-request.bak +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Pull Request -on: - workflow_dispatch: - schedule: - - cron: '0 2 * * *' - -permissions: - contents: write - pull-requests: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v4 - - name: Setup Go - uses: actions/setup-go@v5 - - name: Run go modules tidy - run: | - sudo apt-get install jq - sudo make tidy - sudo make tools.verify.go-gitlint - echo "Run go modules tidy successfully" - continue-on-error: true - - - name: Run go format and lint - run: | - sudo make format - echo "Run go format successfully" - continue-on-error: true - - - name: Run go lint - run: | - sudo make lint - echo "Run go lint successfully" - continue-on-error: true - - - name: Generate all necessary files, such as error code files - run: | - make gen.docgo.doc - make gen - echo "Generate all necessary files successfully" - continue-on-error: true - - - name: make init - run: | - export OPENIM_IP=127.0.0.1 - export LOG_STORAGE_LOCATION="../logs/" - ./scripts/init-config.sh --examples --force - echo "Generate all necessary files successfully" - continue-on-error: true - - - name: Generate Versions Including Pre-release Identifiers - run: | - latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`) - echo $latest_tag > pkg/common/config/version - continue-on-error: true - - - name: Gen CHANGELOG file - run: | - current_tag=$(git describe --tags --abbrev=0) - version=$(echo "$current_tag" | sed -E 's/^v?([0-9]+)\.([0-9]+)\..*$/\1.\2/') - echo "OpenIM Version: $version" - make tools.install.git-chglog - cd CHANGELOG - git-chglog --tag-filter-pattern "v${version}.*" -o CHANGELOG-${version}.md - cd .. - continue-on-error: true - - - name: Run unit test and get test coverage - run: | - make cover - echo "Run unit test and get test coverage successfully" - continue-on-error: true - - - name: OpenIM verify copyright - run: | - sudo make add-copyright - echo "OpenIM verify successfully" - continue-on-error: true - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - commit-message: "cicd: bump League Patch" - author: kubbot <3293172751ysy@gmail.com> - committer: kubbot <3293172751ysy@gmail.com> - # signoff: false - # draft: false - branch: "asf-auto-updates" - assignees: cubxxw - reviewers: cubxxw - title: "[Auto PR 🤖] Bump League Patch auto PR" - body: | - I am a PR generated by robot automation. - - Review criteria: - - - [ ] Disenchanter can connect and issue actions - - Github Actions Status: - - [![Github Pull Request](https://github.com/openimsdk/open-im-server/actions/workflows/pull-request.yml/badge.svg)](https://github.com/openimsdk/open-im-server/actions/workflows/pull-request.yml) - - This is an automated PR. - [workflow](https://github.com/openimsdk/open-im-server/blob/main/.github/workflows/pull-request.yml). - labels: | - kind/documentation - enhancement - report diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index 251f558764..0000000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Release Drafter - -on: - push: - # branches to consider in the event; optional, defaults to all - branches: - - main - # pull_request event is required only for autolabeler - pull_request: - # Only following types are handled by the action, but one can default to all as well - # types: [opened, reopened, synchronize] - # pull_request_target event is required for autolabeler to support PRs from forks - # pull_request_target: - # types: [opened, reopened, synchronize] - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - # write permission is required to create a github release - contents: write - # write permission is required for autolabeler - # otherwise, read permission is required at least - pull-requests: write - runs-on: ubuntu-latest - steps: - # (Optional) GitHub Enterprise requires GHE_HOST variable set - #- name: Set GHE_HOST - # run: | - # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV - - # Drafts your next Release notes as Pull Requests are merged into "master" - - uses: release-drafter/release-drafter@v6 - # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml - # with: - # config-name: my-config.yml - # disable-autolabeler: true - env: - GITHUB_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.bak b/.github/workflows/release.bak deleted file mode 100644 index c15cff6a3c..0000000000 --- a/.github/workflows/release.bak +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Server Release Workflow - -on: - push: - # run only against tags - tags: - - '*' - -permissions: - contents: write - packages: write - issues: write - -jobs: - goreleaser: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - uses: actions/setup-go@v5 - with: - go-version: stable - # More assembly might be required: Docker logins, GPG, etc. It all depends - # on your needs. - - uses: goreleaser/goreleaser-action@v5 - with: - # either 'goreleaser' (default) or 'goreleaser-pro': - distribution: goreleaser - version: latest - workdir: . - args: release -f ./build/goreleaser.yaml --clean --release-footer-tmpl=scripts/template/footer.md.tmpl --release-header-tmpl=scripts/template/head.md.tmpl - env: - USERNAME: ${{ github.repository_owner }} - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - FURY_TOKEN: ${{ secrets.FURY_TOKEN }} - # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' - # distribution: - # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - - goreleaser-check-pkgs: - runs-on: ubuntu-latest - env: - DOCKER_CLI_EXPERIMENTAL: "enabled" - needs: [ goreleaser ] - if: github.ref == 'refs/heads/main' - strategy: - matrix: - format: [ deb, rpm, apk ] - steps: - - uses: actions/checkout@v4 # v3 - with: - fetch-depth: 0 - - uses: arduino/setup-task@e26d8975574116b0097a1161e0fe16ba75d84c1c # v1 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/setup-qemu-action@326560df218a7ea9cf6ab49bbc88b8b306bb437e # v2 - - uses: actions/cache@a2ed59d39b352305bdd2f628719a53b2cc4f9613 # v3 - with: - path: | - ./_output/dist/*.deb - ./_output/dist/*.rpm - ./_output/dist/*.apk - key: ${{ github.ref }} - - run: task goreleaser:test:${{ matrix.format }} diff --git a/.github/workflows/remove-unused-labels.yml b/.github/workflows/remove-unused-labels.yml new file mode 100644 index 0000000000..ab80b1f96e --- /dev/null +++ b/.github/workflows/remove-unused-labels.yml @@ -0,0 +1,74 @@ +name: Remove Unused Labels +on: + workflow_dispatch: + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + contents: read + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Fetch All Issues and PRs + id: fetch_issues_prs + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'all', + per_page: 100 + }); + + const labelsInUse = new Set(); + issues.forEach(issue => { + issue.labels.forEach(label => { + labelsInUse.add(label.name); + }); + }); + + return JSON.stringify(Array.from(labelsInUse)); + result-encoding: string + + - name: Fetch All Labels + id: fetch_labels + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labels = await github.paginate(github.rest.issues.listLabelsForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100 + }); + + return JSON.stringify(labels.map(label => label.name)); + result-encoding: string + + - name: Remove Unused Labels + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelsInUse = new Set(JSON.parse(process.env.LABELS_IN_USE)); + const allLabels = JSON.parse(process.env.ALL_LABELS); + + const unusedLabels = allLabels.filter(label => !labelsInUse.has(label)); + + for (const label of unusedLabels) { + await github.rest.issues.deleteLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label + }); + console.log(`Deleted label: ${label}`); + } + env: + LABELS_IN_USE: ${{ steps.fetch_issues_prs.outputs.result }} + ALL_LABELS: ${{ steps.fetch_labels.outputs.result }} diff --git a/.github/workflows/reopen-issue.yml b/.github/workflows/reopen-issue.yml new file mode 100644 index 0000000000..32f838ba48 --- /dev/null +++ b/.github/workflows/reopen-issue.yml @@ -0,0 +1,78 @@ +name: Reopen and Update Stale Issues + +on: + workflow_dispatch: + +jobs: + reopen_stale_issues: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Fetch Closed Issues with lifecycle/stale Label + id: fetch_issues + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed', + labels: 'lifecycle/stale', + per_page: 100 + }); + const issueNumbers = issues + .filter(issue => !issue.pull_request) // exclude PR + .map(issue => issue.number); + console.log(`Fetched issues: ${issueNumbers}`); + return issueNumbers; + + - name: Set issue numbers + id: set_issue_numbers + run: | + echo "ISSUE_NUMBERS=${{ steps.fetch_issues.outputs.result }}" >> $GITHUB_ENV + echo "Issue numbers: ${{ steps.fetch_issues.outputs.result }}" + + - name: Reopen Issues + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issueNumbers = JSON.parse(process.env.ISSUE_NUMBERS); + console.log(`Reopening issues: ${issueNumbers}`); + + for (const issue_number of issueNumbers) { + // Reopen the issue + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + state: 'open' + }); + console.log(`Reopened issue #${issue_number}`); + } + + - name: Remove lifecycle/stale Label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issueNumbers = JSON.parse(process.env.ISSUE_NUMBERS); + console.log(`Removing 'lifecycle/stale' label from issues: ${issueNumbers}`); + + for (const issue_number of issueNumbers) { + // Remove the lifecycle/stale label + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + name: 'lifecycle/stale' + }); + console.log(`Removed label 'lifecycle/stale' from issue #${issue_number}`); + } diff --git a/.github/workflows/sync-release.bak b/.github/workflows/sync-release.bak deleted file mode 100644 index a85c74fde0..0000000000 --- a/.github/workflows/sync-release.bak +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright © 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -name: Synchronize OpenIM Release Branch Public Code To Other Repositories -on: - push: - paths: - - scripts/* - - docs/* - - config/* - branches: - - release-v*.* - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run GitHub File Sync - uses: BetaHuhn/repo-file-sync-action@latest - with: - GH_INSTALLATION_TOKEN: "${{ secrets.BOT_GITHUB_TOKEN }}" - CONFIG_PATH: .github/sync-release.yml - ORIGINAL_MESSAGE: true - SKIP_PR: true - COMMIT_EACH_FILE: false - COMMIT_BODY: "🤖 kubbot to synchronize the warehouse" - GIT_EMAIL: "3293172751ysy@gmail.com" - GIT_USERNAME: "kubbot" - PR_BODY: 👌 kubecub provides automated community services - REVIEWERS: | - kubbot - cubxxw - PR_LABELS: | - file-sync - automerge - ASSIGNEES: | - kubbot - continue-on-error: true diff --git a/.github/workflows/sync.bak b/.github/workflows/sync.bak deleted file mode 100644 index 595cbbe2c8..0000000000 --- a/.github/workflows/sync.bak +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright © 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -name: Synchronize OpenIM Main Branch Public Code To Other Repositories -on: - push: - branches: - - main - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run GitHub File Sync - uses: BetaHuhn/repo-file-sync-action@latest - with: - GH_INSTALLATION_TOKEN: "${{ secrets.BOT_GITHUB_TOKEN }}" - CONFIG_PATH: .github/sync.yml - ORIGINAL_MESSAGE: true - SKIP_PR: true - COMMIT_EACH_FILE: false - COMMIT_BODY: "🤖 kubbot to synchronize the warehouse" - GIT_EMAIL: "3293172751ysy@gmail.com" - GIT_USERNAME: "kubbot" - PR_BODY: 👌 kubecub provides automated community services - REVIEWERS: | - kubbot - cubxxw - PR_LABELS: | - file-sync - automerge - ASSIGNEES: | - kubbot - continue-on-error: true diff --git a/.github/workflows/greetings.yml b/.github/workflows/user-first-interaction.yml similarity index 65% rename from .github/workflows/greetings.yml rename to .github/workflows/user-first-interaction.yml index b1c85ee375..6999889eb0 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/user-first-interaction.yml @@ -1,18 +1,4 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM First Interaction +name: User First Interaction on: issues: @@ -28,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/first-interaction@v1.3.0 with: - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} + repo-token: ${{ secrets.BOT_TOKEN }} pr-message: | Hello! Thank you for your contribution. diff --git a/.golangci.yml b/.golangci.yml index ae8cea6732..a95e980f85 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,20 +1,3 @@ -# Copyright © 2023 OpenIMSDK open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file contains all available configuration options -# with their default values. - # options for analysis running run: # default concurrency is a available CPU number @@ -302,7 +285,7 @@ linters-settings: gofumpt: # Select the Go version to target. The default is `1.18`. - lang-version: "1.20" + go-version: "1.21" # Choose whether or not to use the extra rules that are disabled # by default From 27024dfb032ab9015b2779c26a7a70d8239eb859 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 14 Aug 2024 18:20:42 +0800 Subject: [PATCH 59/91] fix: solve uncorrect outdated msg get. (#2513) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. --- .github/workflows/help-comment-issue.yml | 2 +- internal/rpc/conversation/conversaion.go | 4 +-- internal/rpc/msg/clear.go | 27 +++++++------- internal/tools/cron_task.go | 1 + pkg/common/storage/controller/msg.go | 36 ++++++++++++++++--- pkg/common/storage/database/mgo/msg.go | 46 +++++++++++++++++++++++- pkg/common/storage/database/msg.go | 7 ++-- 7 files changed, 99 insertions(+), 24 deletions(-) diff --git a/.github/workflows/help-comment-issue.yml b/.github/workflows/help-comment-issue.yml index c4e72ffc67..b1cc621828 100644 --- a/.github/workflows/help-comment-issue.yml +++ b/.github/workflows/help-comment-issue.yml @@ -29,7 +29,7 @@ jobs: uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.issue.number }} - token: ${{ secrets.BOT_GITHUB_TOKEN }} + token: ${{ secrets.BOT_TOKEN }} body: | This issue is available for anyone to work on. **Make sure to reference this issue in your pull request.** :sparkles: Thank you for your contribution! :sparkles: [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) to connect and communicate with our developers. diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 4cf20f919c..117ed8ca71 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -634,11 +634,11 @@ func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Contex conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) if err != nil { - log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) + // log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) continue } - log.ZDebug(ctx, "PageConversationIDs success", "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) + // log.ZDebug(ctx, "PageConversationIDs success", "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) if len(conversationIDs) == 0 { continue } diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 6be551eada..4ffa1f43ee 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -30,8 +30,14 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. msgNum int start = time.Now() ) + clearMsg := func(ctx context.Context) (bool, error) { - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, 100) + docIDs, err := m.MsgDatabase.GetDocIDs(ctx) + if err != nil { + return false, err + } + + msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, 5000) if err != nil { return false, err } @@ -55,19 +61,14 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return true, nil } - for { - keep, err := clearMsg(ctx) - if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return nil, err - } - if !keep { - log.ZInfo(ctx, "clear msg success", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - break - } - - log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + _, err = clearMsg(ctx) + if err != nil { + log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + return nil, err } + + log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + return &msg.ClearMsgResp{}, nil } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index b1d59800ce..afbaf34b43 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -81,6 +81,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 49268e0493..caa491a741 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "strings" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -97,8 +98,10 @@ type CommonMsgDatabase interface { ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // clear msg - GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) + GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + + GetDocIDs(ctx context.Context) ([]string, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -912,8 +915,25 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { - return db.msgDocDatabase.GetBeforeMsg(ctx, ts, limit) +func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { + var msgs []*model.MsgDocModel + for i := 0; i < len(docIDs); i += 1000 { + end := i + 1000 + if end > len(docIDs) { + end = len(docIDs) + } + + res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) + if err != nil { + return nil, err + } + msgs = append(msgs, res...) + + if len(msgs) >= limit { + return msgs[:limit], nil + } + } + return msgs, nil } func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { @@ -936,8 +956,10 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d return index, err } if len(index) == notNull { + log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) } else { + log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) } } @@ -955,3 +977,7 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin } return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } + +func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { + return db.msgDocDatabase.GetDocIDs(ctx) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 7dc308a7c4..7b45cdf51e 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -8,6 +8,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/utils/datautil" + "golang.org/x/exp/rand" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -1226,10 +1227,53 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { + limit := 5000 + var skip int + var docIDs []string + var offset int + + count, err := m.coll.CountDocuments(ctx, bson.M{}) + if err != nil { + return nil, err + } + + if count < int64(limit) { + skip = 0 + } else { + rand.Seed(uint64(time.Now().UnixMilli())) + skip = rand.Intn(int(count / int64(limit))) + offset = skip * limit + } + log.ZDebug(ctx, "offset", "skip", skip, "offset", offset) + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ + { + "$project": bson.M{ + "doc_id": 1, + }, + }, + { + "$skip": offset, + }, + { + "$limit": limit, + }, + }) + + for _, doc := range res { + docIDs = append(docIDs, doc.DocID) + } + + return docIDs, errs.Wrap(err) +} + +func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ + "doc_id": bson.M{ + "$in": docIDs, + }, "msgs.msg.send_time": bson.M{ "$lt": ts, }, diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 84f3a9e3e2..23a99f5b96 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -16,10 +16,11 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/msg" "go.mongodb.org/mongo-driver/mongo" - "time" ) type Msg interface { @@ -44,5 +45,7 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) + GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + + GetDocIDs(ctx context.Context) ([]string, error) } From 6f901bc3899b7784a1b78c86058519516f54e406 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 14 Aug 2024 18:27:09 +0800 Subject: [PATCH 60/91] feat: update issue translator in workflows (#2521) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * feat: update issue translator. --- .github/workflows/issue-translator.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/issue-translator.yml diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml new file mode 100644 index 0000000000..6a8528ae62 --- /dev/null +++ b/.github/workflows/issue-translator.yml @@ -0,0 +1,19 @@ +name: 'issue-translator' +on: + issue_comment: + types: [created] + issues: + types: [opened] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: usthe/issues-translate-action@v2.7 + with: + BOT_GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} + IS_MODIFY_TITLE: true + # not require, default false, . Decide whether to modify the issue title + # if true, the robot account @Issues-translate-bot must have modification permissions, invite @Issues-translate-bot to your project or use your custom bot. + CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically. 👯👭🏻🧑‍🤝‍🧑👫🧑🏿‍🤝‍🧑🏻👩🏾‍🤝‍👨🏿👬🏿 + # not require. Customize the translation robot prefix message. \ No newline at end of file From 188507a8acf3064589df5cd2d3972335d91750de Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 19 Aug 2024 16:13:48 +0800 Subject: [PATCH 61/91] fix: pass getMinioImageThumbnailKey error. (#2532) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * feat: update issue translator. * fix: pass getMinioImageThumbnailKey error. --- internal/rpc/third/s3.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index f96eb73905..fb6a1157e1 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -290,6 +290,7 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) + var deltotal int findPagination := &sdkws.RequestPagination{ PageNumber: 1, ShowNumber: 1000, @@ -311,10 +312,8 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO return nil, errs.Wrap(err) } if int(count) < 1 && t.minio != nil { - thumbnailKey, err := t.getMinioImageThumbnailKey(ctx, key) - if err != nil { - return nil, errs.Wrap(err) - } + thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) + t.s3dataBase.DeleteObject(ctx, thumbnailKey) t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) t.s3dataBase.DeleteObject(ctx, key) @@ -329,7 +328,9 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if total < int64(findPagination.ShowNumber) { break } + deltotal += int(total) } + log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", deltotal) return &third.DeleteOutdatedDataResp{}, nil } From 0b06e204642ff4b76c54e28aee713895565980d7 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 19 Aug 2024 19:14:24 +0800 Subject: [PATCH 62/91] docs: update CLA comments contents. (#2534) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * docs: update CLA comments contents. * update workflow file. --- .github/workflows/go-build-test.yml | 3 +++ CONTRIBUTING-zh_CN.md | 2 +- CONTRIBUTING.md | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 14546f4f45..5341c919d6 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -7,6 +7,9 @@ on: pull_request: branches: - main + paths-ignore: + - '**/*.md' + workflow_dispatch: jobs: diff --git a/CONTRIBUTING-zh_CN.md b/CONTRIBUTING-zh_CN.md index 47965a9f49..2f875b1405 100644 --- a/CONTRIBUTING-zh_CN.md +++ b/CONTRIBUTING-zh_CN.md @@ -77,7 +77,7 @@ git push origin fix-bug-123 ### 9. 签署 CLA 如果这是你第一次提交 PR，你需要在 PR 的评论中回复： ``` -I have read the CLA Document and I hereby sign the CLA +The signature to be committed in order to sign the CLA ``` ### 编程规范 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aa07393ee..6ed419b2f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,7 +75,7 @@ Go to your fork on GitHub and click the "Pull Request" button. Ensure the PR des ### 9. Sign the CLA If this is your first time submitting a PR, you will need to reply in the comments of the PR: ``` -I have read the CLA Document and I hereby sign the CLA +The signature to be committed in order to sign the CLA ``` ### Programming Standards From 7f6b4da8eb207f0e71a2045441f930546ca5c340 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:27:47 +0800 Subject: [PATCH 63/91] fix: the log key value is not aligned (#2527) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even --------- Co-authored-by: withchao --- internal/msggateway/ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 537b8c5f0a..9b18ade7dc 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -275,7 +275,7 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} - log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", ws.msgGatewayConfig.Discovery.Enable) + log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", "discoveryEnable", ws.msgGatewayConfig.Discovery.Enable) if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) From 1022b297d45e7701f8f38dc470705f75e51af20e Mon Sep 17 00:00:00 2001 From: Mew151 Date: Wed, 21 Aug 2024 11:44:00 +0800 Subject: [PATCH 64/91] =?UTF-8?q?fix=20=E7=BB=86=E8=8A=82=20(#2525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、统一结构体方法 receiver，都用 pointer 2、使用 errors.Is 来做错误判断 3、修复单词拼写的错误 --- internal/api/msg.go | 6 +++--- internal/msgtransfer/init.go | 3 ++- .../msgtransfer/online_history_msg_handler.go | 3 ++- .../msgtransfer/online_msg_to_mongo_handler.go | 6 +++--- internal/push/onlinepusher.go | 16 ++++++++-------- internal/push/push_handler.go | 8 ++++---- pkg/common/storage/controller/msg.go | 4 ++-- pkg/common/storage/database/mgo/msg.go | 6 +++--- pkg/common/storage/model/msg.go | 16 ++++++++-------- 9 files changed, 35 insertions(+), 33 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index ba63fbb66f..bf7cb83a43 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -49,14 +49,14 @@ func NewMessageApi(msgRpcClient *rpcclient.Message, userRpcClient *rpcclient.Use userRpcClient: rpcclient.NewUserRpcClientByUser(userRpcClient), imAdminUserID: imAdminUserID} } -func (MessageApi) SetOptions(options map[string]bool, value bool) { +func (*MessageApi) SetOptions(options map[string]bool, value bool) { datautil.SetSwitchFromOptions(options, constant.IsHistory, value) datautil.SetSwitchFromOptions(options, constant.IsPersistent, value) datautil.SetSwitchFromOptions(options, constant.IsSenderSync, value) datautil.SetSwitchFromOptions(options, constant.IsConversationUpdate, value) } -func (m MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) *msg.SendMsgReq { +func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) *msg.SendMsgReq { var newContent string options := make(map[string]bool, 5) switch params.ContentType { @@ -231,7 +231,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { } // Set the status to successful if the message is sent. - var status int = constant.MsgSendSuccessed + var status = constant.MsgSendSuccessed // Attempt to update the message sending status in the system. _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index b4b2245eb0..5e0ccd0e52 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -16,6 +16,7 @@ package msgtransfer import ( "context" + "errors" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -137,7 +138,7 @@ func (m *MsgTransfer) Start(index int, config *Config) error { return } - if err := prommetrics.TransferInit(prometheusPort); err != nil && err != http.ErrServerClosed { + if err := prommetrics.TransferInit(prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index d671ec52a2..77161202cb 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -16,6 +16,7 @@ package msgtransfer import ( "context" + "errors" "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -187,7 +188,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key if len(storageMessageList) > 0 { msg := storageMessageList[0] lastSeq, isNewConversation, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index e5651012c6..a72fb4792b 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -91,13 +91,13 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont } } -func (OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } -func (OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, -) error { // a instance in the consumer group +) error { // an instance in the consumer group log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) for msg := range claim.Messages() { diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index a61399fb6b..d0c65e06bf 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -19,20 +19,20 @@ type OnlinePusher interface { pushToUserIDs *[]string) []string } -type emptyOnlinePUsher struct{} +type emptyOnlinePusher struct{} -func newEmptyOnlinePUsher() *emptyOnlinePUsher { - return &emptyOnlinePUsher{} +func newEmptyOnlinePusher() *emptyOnlinePusher { + return &emptyOnlinePusher{} } -func (emptyOnlinePUsher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, +func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - log.ZWarn(ctx, "emptyOnlinePUsher GetConnsAndOnlinePush", nil) + log.ZWarn(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) return nil, nil } -func (u emptyOnlinePUsher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, +func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { - log.ZWarn(ctx, "emptyOnlinePUsher GetOnlinePushFailedUserIDs", nil) + log.ZWarn(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) return nil } @@ -45,7 +45,7 @@ func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) Onli case "etcd": return NewDefaultAllNode(disCov, config) default: - return newEmptyOnlinePUsher() + return newEmptyOnlinePusher() } } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 249622a592..8ecb3dad1a 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -154,17 +154,17 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * return nil } } - offlinePUshUserID := []string{msg.RecvID} + offlinePushUserID := []string{msg.RecvID} //receiver offline push if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, - offlinePUshUserID, msg, nil); err != nil { + offlinePushUserID, msg, nil); err != nil { return err } - err = c.offlinePushMsg(ctx, msg, offlinePUshUserID) + err = c.offlinePushMsg(ctx, msg, offlinePushUserID) if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePUshUserID", offlinePUshUserID, "msg", msg) + log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) return nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index caa491a741..8eb417f939 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -90,8 +90,8 @@ type CommonMsgDatabase interface { // to mq MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error - MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error) - MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error + MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) + MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 7b45cdf51e..fc1fe47eab 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -118,9 +118,9 @@ func (m *MsgMgo) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID strin } func (m *MsgMgo) getMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { - indexs := make([]int64, 0, len(seqs)) + indexes := make([]int64, 0, len(seqs)) for _, seq := range seqs { - indexs = append(indexs, m.model.GetMsgIndex(seq)) + indexes = append(indexes, m.model.GetMsgIndex(seq)) } pipeline := mongo.Pipeline{ bson.D{{Key: "$match", Value: bson.D{ @@ -131,7 +131,7 @@ func (m *MsgMgo) getMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID strin {Key: "doc_id", Value: 1}, {Key: "msgs", Value: bson.D{ {Key: "$map", Value: bson.D{ - {Key: "input", Value: indexs}, + {Key: "input", Value: indexes}, {Key: "as", Value: "index"}, {Key: "in", Value: bson.D{ {Key: "$arrayElemAt", Value: bson.A{"$msgs", "$$index"}}, diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 8095665d2f..e16233973b 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -92,15 +92,15 @@ type GroupCount struct { Count int64 `bson:"count"` } -func (MsgDocModel) TableName() string { +func (*MsgDocModel) TableName() string { return MsgTableName } -func (MsgDocModel) GetSingleGocMsgNum() int64 { +func (*MsgDocModel) GetSingleGocMsgNum() int64 { return singleGocMsgNum } -func (MsgDocModel) GetSingleGocMsgNum5000() int64 { +func (*MsgDocModel) GetSingleGocMsgNum5000() int64 { return singleGocMsgNum5000 } @@ -108,12 +108,12 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } -func (m MsgDocModel) GetDocID(conversationID string, seq int64) string { +func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) } -func (m MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 { +func (m *MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 { t := make(map[string][]int64) for i := 0; i < len(seqs); i++ { docID := m.GetDocID(conversationID, seqs[i]) @@ -127,15 +127,15 @@ func (m MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[st return t } -func (MsgDocModel) GetMsgIndex(seq int64) int64 { +func (*MsgDocModel) GetMsgIndex(seq int64) int64 { return (seq - 1) % singleGocMsgNum } -func (MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { +func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } -func (MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { +func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) msgModel.Seq = v From 0af207b625a907c5218be557ce26a04bf96540bb Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:10:43 +0800 Subject: [PATCH 65/91] Fix config (#2541) * feat: remove double quotation marks * feat: test * feat: default close prometheus --- config/discovery.yml | 2 +- config/kafka.yml | 18 +-- config/minio.yml | 12 +- config/mongodb.yml | 2 +- config/notification.yml | 188 +++++++++++++------------- config/openim-crontask.yml | 2 +- config/openim-msggateway.yml | 2 +- config/openim-push.yml | 32 ++--- config/openim-rpc-auth.yml | 2 +- config/openim-rpc-conversation.yml | 2 +- config/openim-rpc-friend.yml | 2 +- config/openim-rpc-group.yml | 2 +- config/openim-rpc-msg.yml | 2 +- config/openim-rpc-third.yml | 34 ++--- config/openim-rpc-user.yml | 2 +- config/prometheus.yml | 82 +++++------ config/redis.yml | 2 +- config/share.yml | 2 +- config/webhooks.yml | 2 +- docker-compose.yml | 90 ++++++------ pkg/common/config/load_config_test.go | 23 ++++ 21 files changed, 264 insertions(+), 241 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 3d96ff9b66..78a36f3d1f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,4 +1,4 @@ -enable: "etcd" +enable: etcd etcd: rootDirectory: openim address: [ localhost:12379 ] diff --git a/config/kafka.yml b/config/kafka.yml index d412e1be06..e45474f275 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -3,17 +3,17 @@ username: '' # Password for authentication password: '' # Producer acknowledgment settings -producerAck: "" +producerAck: # Compression type to use (e.g., none, gzip, snappy) -compressType: "none" +compressType: none # List of Kafka broker addresses address: [ localhost:19094 ] # Kafka topic for Redis integration -toRedisTopic: "toRedis" +toRedisTopic: toRedis # Kafka topic for MongoDB integration -toMongoTopic: "toMongo" +toMongoTopic: toMongo # Kafka topic for push notifications -toPushTopic: "toPush" +toPushTopic: toPush # Consumer group ID for Redis topic toRedisGroupID: redis # Consumer group ID for MongoDB topic @@ -25,12 +25,12 @@ tls: # Enable or disable TLS enableTLS: false # CA certificate file path - caCrt: "" + caCrt: # Client certificate file path - clientCrt: "" + clientCrt: # Client key file path - clientKey: "" + clientKey: # Client key password - clientKeyPwd: "" + clientKeyPwd: # Whether to skip TLS verification (not recommended for production) insecureSkipVerify: false diff --git a/config/minio.yml b/config/minio.yml index 11a9ace354..ad1a32a8c2 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -1,15 +1,15 @@ # Name of the bucket in MinIO -bucket: "openim" +bucket: openim # Access key ID for MinIO authentication -accessKeyID: "root" +accessKeyID: root # Secret access key for MinIO authentication -secretAccessKey: "openIM123" +secretAccessKey: openIM123 # Session token for MinIO authentication (optional) -sessionToken: '' +sessionToken: # Internal address of the MinIO server -internalAddress: "localhost:10005" +internalAddress: localhost:10005 # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://external_ip:10005" +externalAddress: http://external_ip:10005 # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 98f5694e45..78f85992c9 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,5 +1,5 @@ # URI for database connection, leave empty if using address and credential settings directly -uri: '' +uri: # List of MongoDB server addresses address: [ localhost:37017 ] # Name of the database diff --git a/config/notification.yml b/config/notification.yml index 278376c244..85ca91af18 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -28,11 +28,11 @@ groupCreated: # Enables or disables offline push notifications. enable: false # Title for the notification when a group is created. - title: "create group title" + title: create group title # Description for the notification. - desc: "create group desc" + desc: create group desc # Additional information for the notification. - ext: "create group ext" + ext: create group ext groupInfoSet: isSendMsg: false @@ -40,9 +40,9 @@ groupInfoSet: unreadCount: false offlinePush: enable: false - title: "groupInfoSet title" - desc: "groupInfoSet desc" - ext: "groupInfoSet ext" + title: groupInfoSet title + desc: groupInfoSet desc + ext: groupInfoSet ext joinGroupApplication: @@ -51,9 +51,9 @@ joinGroupApplication: unreadCount: false offlinePush: enable: false - title: "joinGroupApplication title" - desc: "joinGroupApplication desc" - ext: "joinGroupApplication ext" + title: joinGroupApplication title + desc: joinGroupApplication desc + ext: joinGroupApplication ext memberQuit: isSendMsg: true @@ -61,9 +61,9 @@ memberQuit: unreadCount: false offlinePush: enable: false - title: "memberQuit title" - desc: "memberQuit desc" - ext: "memberQuit ext" + title: memberQuit title + desc: memberQuit desc + ext: memberQuit ext groupApplicationAccepted: isSendMsg: false @@ -71,9 +71,9 @@ groupApplicationAccepted: unreadCount: false offlinePush: enable: false - title: "groupApplicationAccepted title" - desc: "groupApplicationAccepted desc" - ext: "groupApplicationAccepted ext" + title: groupApplicationAccepted title + desc: groupApplicationAccepted desc + ext: groupApplicationAccepted ext groupApplicationRejected: isSendMsg: false @@ -81,9 +81,9 @@ groupApplicationRejected: unreadCount: false offlinePush: enable: false - title: "groupApplicationRejected title" - desc: "groupApplicationRejected desc" - ext: "groupApplicationRejected ext" + title: groupApplicationRejected title + desc: groupApplicationRejected desc + ext: groupApplicationRejected ext groupOwnerTransferred: @@ -92,9 +92,9 @@ groupOwnerTransferred: unreadCount: false offlinePush: enable: false - title: "groupOwnerTransferred title" - desc: "groupOwnerTransferred desc" - ext: "groupOwnerTransferred ext" + title: groupOwnerTransferred title + desc: groupOwnerTransferred desc + ext: groupOwnerTransferred ext memberKicked: isSendMsg: true @@ -102,9 +102,9 @@ memberKicked: unreadCount: false offlinePush: enable: false - title: "memberKicked title" - desc: "memberKicked desc" - ext: "memberKicked ext" + title: memberKicked title + desc: memberKicked desc + ext: memberKicked ext memberInvited: isSendMsg: true @@ -112,9 +112,9 @@ memberInvited: unreadCount: false offlinePush: enable: false - title: "memberInvited title" - desc: "memberInvited desc" - ext: "memberInvited ext" + title: memberInvited title + desc: memberInvited desc + ext: memberInvited ext memberEnter: isSendMsg: true @@ -122,9 +122,9 @@ memberEnter: unreadCount: false offlinePush: enable: false - title: "memberEnter title" - desc: "memberEnter desc" - ext: "memberEnter ext" + title: memberEnter title + desc: memberEnter desc + ext: memberEnter ext groupDismissed: isSendMsg: true @@ -132,9 +132,9 @@ groupDismissed: unreadCount: false offlinePush: enable: false - title: "groupDismissed title" - desc: "groupDismissed desc" - ext: "groupDismissed ext" + title: groupDismissed title + desc: groupDismissed desc + ext: groupDismissed ext groupMuted: isSendMsg: true @@ -142,9 +142,9 @@ groupMuted: unreadCount: false offlinePush: enable: false - title: "groupMuted title" - desc: "groupMuted desc" - ext: "groupMuted ext" + title: groupMuted title + desc: groupMuted desc + ext: groupMuted ext groupCancelMuted: isSendMsg: true @@ -152,11 +152,11 @@ groupCancelMuted: unreadCount: false offlinePush: enable: false - title: "groupCancelMuted title" - desc: "groupCancelMuted desc" - ext: "groupCancelMuted ext" + title: groupCancelMuted title + desc: groupCancelMuted desc + ext: groupCancelMuted ext defaultTips: - tips: "group Cancel Muted" + tips: group Cancel Muted groupMemberMuted: @@ -165,9 +165,9 @@ groupMemberMuted: unreadCount: false offlinePush: enable: false - title: "groupMemberMuted title" - desc: "groupMemberMuted desc" - ext: "groupMemberMuted ext" + title: groupMemberMuted title + desc: groupMemberMuted desc + ext: groupMemberMuted ext groupMemberCancelMuted: isSendMsg: true @@ -175,9 +175,9 @@ groupMemberCancelMuted: unreadCount: false offlinePush: enable: false - title: "groupMemberCancelMuted title" - desc: "groupMemberCancelMuted desc" - ext: "groupMemberCancelMuted ext" + title: groupMemberCancelMuted title + desc: groupMemberCancelMuted desc + ext: groupMemberCancelMuted ext groupMemberInfoSet: isSendMsg: false @@ -185,9 +185,9 @@ groupMemberInfoSet: unreadCount: false offlinePush: enable: false - title: "groupMemberInfoSet title" - desc: "groupMemberInfoSet desc" - ext: "groupMemberInfoSet ext" + title: groupMemberInfoSet title + desc: groupMemberInfoSet desc + ext: groupMemberInfoSet ext groupInfoSetAnnouncement: isSendMsg: true @@ -195,9 +195,9 @@ groupInfoSetAnnouncement: unreadCount: false offlinePush: enable: false - title: "groupInfoSetAnnouncement title" - desc: "groupInfoSetAnnouncement desc" - ext: "groupInfoSetAnnouncement ext" + title: groupInfoSetAnnouncement title + desc: groupInfoSetAnnouncement desc + ext: groupInfoSetAnnouncement ext groupInfoSetName: @@ -206,9 +206,9 @@ groupInfoSetName: unreadCount: false offlinePush: enable: false - title: "groupInfoSetName title" - desc: "groupInfoSetName desc" - ext: "groupInfoSetName ext" + title: groupInfoSetName title + desc: groupInfoSetName desc + ext: groupInfoSetName ext #############################friend################################# @@ -218,9 +218,9 @@ friendApplicationAdded: unreadCount: false offlinePush: enable: false - title: "Somebody applies to add you as a friend" - desc: "Somebody applies to add you as a friend" - ext: "Somebody applies to add you as a friend" + title: Somebody applies to add you as a friend + desc: Somebody applies to add you as a friend + ext: Somebody applies to add you as a friend friendApplicationApproved: isSendMsg: true @@ -228,9 +228,9 @@ friendApplicationApproved: unreadCount: false offlinePush: enable: true - title: "Someone applies to add your friend application" - desc: "Someone applies to add your friend application" - ext: "Someone applies to add your friend application" + title: Someone applies to add your friend application + desc: Someone applies to add your friend application + ext: Someone applies to add your friend application friendApplicationRejected: isSendMsg: false @@ -238,9 +238,9 @@ friendApplicationRejected: unreadCount: false offlinePush: enable: true - title: "Someone rejected your friend application" - desc: "Someone rejected your friend application" - ext: "Someone rejected your friend application" + title: Someone rejected your friend application + desc: Someone rejected your friend application + ext: Someone rejected your friend application friendAdded: isSendMsg: false @@ -248,9 +248,9 @@ friendAdded: unreadCount: false offlinePush: enable: true - title: "We have become friends" - desc: "We have become friends" - ext: "We have become friends" + title: We have become friends + desc: We have become friends + ext: We have become friends friendDeleted: isSendMsg: false @@ -258,9 +258,9 @@ friendDeleted: unreadCount: false offlinePush: enable: true - title: "deleted a friend" - desc: "deleted a friend" - ext: "deleted a friend" + title: deleted a friend + desc: deleted a friend + ext: deleted a friend friendRemarkSet: isSendMsg: false @@ -268,9 +268,9 @@ friendRemarkSet: unreadCount: false offlinePush: enable: true - title: "Your friend's profile has been changed" - desc: "Your friend's profile has been changed" - ext: "Your friend's profile has been changed" + title: Your friend's profile has been changed + desc: Your friend's profile has been changed + ext: Your friend's profile has been changed blackAdded: isSendMsg: false @@ -278,9 +278,9 @@ blackAdded: unreadCount: false offlinePush: enable: true - title: "blocked a user" - desc: "blocked a user" - ext: "blocked a user" + title: blocked a user + desc: blocked a user + ext: blocked a user blackDeleted: isSendMsg: false @@ -288,9 +288,9 @@ blackDeleted: unreadCount: false offlinePush: enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" + title: Remove a blocked user + desc: Remove a blocked user + ext: Remove a blocked user friendInfoUpdated: isSendMsg: false @@ -298,9 +298,9 @@ friendInfoUpdated: unreadCount: false offlinePush: enable: true - title: "friend info updated" - desc: "friend info updated" - ext: "friend info updated" + title: friend info updated + desc: friend info updated + ext: friend info updated #####################user######################### userInfoUpdated: @@ -309,9 +309,9 @@ userInfoUpdated: unreadCount: false offlinePush: enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" + title: Remove a blocked user + desc: Remove a blocked user + ext: Remove a blocked user userStatusChanged: isSendMsg: false @@ -319,9 +319,9 @@ userStatusChanged: unreadCount: false offlinePush: enable: false - title: "user status changed" - desc: "user status changed" - ext: "user status changed" + title: user status changed + desc: user status changed + ext: user status changed #####################conversation######################### conversationChanged: @@ -330,9 +330,9 @@ conversationChanged: unreadCount: false offlinePush: enable: true - title: "conversation changed" - desc: "conversation changed" - ext: "conversation changed" + title: conversation changed + desc: conversation changed + ext: conversation changed conversationSetPrivate: isSendMsg: true @@ -340,6 +340,6 @@ conversationSetPrivate: unreadCount: false offlinePush: enable: true - title: "burn after reading" - desc: "burn after reading" - ext: "burn after reading" + title: burn after reading + desc: burn after reading + ext: burn after reading diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 3839104a44..c05bd2485f 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,3 +1,3 @@ -cronExecuteTime: "0 2 * * *" +cronExecuteTime: 0 2 * * * retainChatRecords: 365 fileExpireTime: 90 diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 0c92d83278..d8068c4435 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports ports: [ 10140 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 9384008a04..1e55cdee87 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports @@ -13,28 +13,28 @@ prometheus: ports: [ 20107 ] maxConcurrentWorkers: 3 -#"Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified." -enable: "geTui" +#Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. +enable: geTui geTui: - pushUrl: "https://restapi.getui.com/v2/$appId" - masterSecret: '' - appKey: '' - intent: '' - channelID: '' - channelName: '' + pushUrl: https://restapi.getui.com/v2/$appId + masterSecret: + appKey: + intent: + channelID: + channelName: fcm: # Prioritize using file paths. If the file path is empty, use URL - filePath: "" # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. - authURL: "" # Must start with https or http. + filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. + authURL: # Must start with https or http. jpns: - appKey: '' - masterSecret: '' - pushURL: '' - pushIntent: '' + appKey: + masterSecret: + pushURL: + pushIntent: # iOS system push sound and badge count iosPush: - pushSound: "xxx" + pushSound: xxx badgeCount: true production: false diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 2d861cd5ab..979eca06f4 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index a094bfac10..d3f8225019 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 7b829f971c..8d5869ce81 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 78b44030e0..f49c46207f 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 17ce26e9b2..38cc46ecfc 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 6fb60f47f8..408251f4df 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports @@ -15,26 +15,26 @@ prometheus: object: # Use MinIO as object storage, or set to "cos", "oss", "kodo", "aws", while also configuring the corresponding settings - enable: "minio" + enable: minio cos: bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com - secretID: '' - secretKey: '' - sessionToken: '' + secretID: + secretKey: + sessionToken: publicRead: false oss: - endpoint: "https://oss-cn-chengdu.aliyuncs.com" - bucket: "demo-9999999" - bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" - accessKeyID: '' - accessKeySecret: '' - sessionToken: '' + endpoint: https://oss-cn-chengdu.aliyuncs.com + bucket: demo-9999999 + bucketURL: https://demo-9999999.oss-cn-chengdu.aliyuncs.com + accessKeyID: + accessKeySecret: + sessionToken: publicRead: false kodo: - endpoint: "http://s3.cn-south-1.qiniucs.com" - bucket: "kodo-bucket-test" - bucketURL: "http://kodo-bucket-test-oetobfb.qiniudns.com" - accessKeyID: '' - accessKeySecret: '' - sessionToken: '' + endpoint: http://s3.cn-south-1.qiniucs.com + bucket: kodo-bucket-test + bucketURL: http://kodo-bucket-test-oetobfb.qiniudns.com + accessKeyID: + accessKeySecret: + sessionToken: publicRead: false \ No newline at end of file diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index cbfb55b6c7..ec9bb30dcb 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -1,6 +1,6 @@ rpc: # API or other RPCs can access this RPC through this IP; if left blank, the internal network IP is obtained by default - registerIP: '' + registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports diff --git a/config/prometheus.yml b/config/prometheus.yml index 5db41679f4..c7ce4a489c 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -8,76 +8,76 @@ global: alerting: alertmanagers: - static_configs: - - targets: ['internal_ip:19093'] + - targets: [internal_ip:19093] -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +# Load rules once and periodically evaluate them according to the global evaluation_interval. rule_files: - - "instance-down-rules.yml" -# - "first_rules.yml" -# - "second_rules.yml" + - instance-down-rules.yml +# - first_rules.yml +# - second_rules.yml # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: - # The job name is added as a label "job='job_name'"" to any timeseries scraped from this config. + # The job name is added as a label job=job_name" to any timeseries scraped from this config. # Monitored information captured by prometheus # prometheus fetches application services - - job_name: 'node_exporter' + - job_name: node_exporter static_configs: - - targets: [ 'internal_ip:20114' ] - - job_name: 'openimserver-openim-api' + - targets: [ internal_ip:20114 ] + - job_name: openimserver-openim-api static_configs: - - targets: [ 'internal_ip:20113' ] + - targets: [ internal_ip:20113 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-msggateway' + namespace: default + - job_name: openimserver-openim-msggateway static_configs: - - targets: [ 'internal_ip:20112' ] + - targets: [ internal_ip:20112 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-msgtransfer' + namespace: default + - job_name: openimserver-openim-msgtransfer static_configs: - - targets: [ 'internal_ip:20111', 'internal_ip:20110', 'internal_ip:20109', 'internal_ip:20108' ] + - targets: [ internal_ip:20111, internal_ip:20110, internal_ip:20109, internal_ip:20108 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-push' + namespace: default + - job_name: openimserver-openim-push static_configs: - - targets: [ 'internal_ip:20107' ] + - targets: [ internal_ip:20107 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-auth' + namespace: default + - job_name: openimserver-openim-rpc-auth static_configs: - - targets: [ 'internal_ip:20106' ] + - targets: [ internal_ip:20106 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-conversation' + namespace: default + - job_name: openimserver-openim-rpc-conversation static_configs: - - targets: [ 'internal_ip:20105' ] + - targets: [ internal_ip:20105 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-friend' + namespace: default + - job_name: openimserver-openim-rpc-friend static_configs: - - targets: [ 'internal_ip:20104' ] + - targets: [ internal_ip:20104 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-group' + namespace: default + - job_name: openimserver-openim-rpc-group static_configs: - - targets: [ 'internal_ip:20103' ] + - targets: [ internal_ip:20103 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-msg' + namespace: default + - job_name: openimserver-openim-rpc-msg static_configs: - - targets: [ 'internal_ip:20102' ] + - targets: [ internal_ip:20102 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-third' + namespace: default + - job_name: openimserver-openim-rpc-third static_configs: - - targets: [ 'internal_ip:20101' ] + - targets: [ internal_ip:20101 ] labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-user' + namespace: default + - job_name: openimserver-openim-rpc-user static_configs: - - targets: [ 'internal_ip:20100' ] + - targets: [ internal_ip:20100 ] labels: - namespace: 'default' \ No newline at end of file + namespace: default \ No newline at end of file diff --git a/config/redis.yml b/config/redis.yml index 87abed0e1c..83e3054590 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,5 +1,5 @@ address: [ localhost:16379 ] -username: '' +username: password: openIM123 clusterMode: false db: 0 diff --git a/config/share.yml b/config/share.yml index fc97b6a1ff..4c5892615b 100644 --- a/config/share.yml +++ b/config/share.yml @@ -10,5 +10,5 @@ rpcRegisterName: conversation: conversation third: third -imAdminUserID: [ "imAdmin" ] +imAdminUserID: [ imAdmin ] diff --git a/config/webhooks.yml b/config/webhooks.yml index 11a85ba0c4..421522ff86 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -1,4 +1,4 @@ -url: "webhook://127.0.0.1:10008/callbackExample" +url: webhook://127.0.0.1:10008/callbackExample beforeSendSingleMsg: enable: false timeout: 5 diff --git a/docker-compose.yml b/docker-compose.yml index 8cc1f24b2f..49c44d3d9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,50 +140,50 @@ services: networks: - openim - prometheus: - image: ${PROMETHEUS_IMAGE} - container_name: prometheus - restart: always - volumes: - - ./config/prometheus.yml:/etc/prometheus/prometheus.yml - - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml - - ${DATA_DIR}/components/prometheus/data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - ports: - - "19091:9090" - networks: - - openim - - alertmanager: - image: ${ALERTMANAGER_IMAGE} - container_name: alertmanager - restart: always - volumes: - - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - - ./config/email.tmpl:/etc/alertmanager/email.tmpl - ports: - - "19093:9093" - networks: - - openim - - grafana: - image: ${GRAFANA_IMAGE} - container_name: grafana - user: root - restart: always - environment: - - GF_SECURITY_ALLOW_EMBEDDING=true - - GF_SESSION_COOKIE_SAMESITE=none - - GF_SESSION_COOKIE_SECURE=true - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - ports: - - "13000:3000" - volumes: - - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana - networks: - - openim +# prometheus: +# image: ${PROMETHEUS_IMAGE} +# container_name: prometheus +# restart: always +# volumes: +# - ./config/prometheus.yml:/etc/prometheus/prometheus.yml +# - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml +# - ${DATA_DIR}/components/prometheus/data:/prometheus +# command: +# - '--config.file=/etc/prometheus/prometheus.yml' +# - '--storage.tsdb.path=/prometheus' +# ports: +# - "19091:9090" +# networks: +# - openim +# +# alertmanager: +# image: ${ALERTMANAGER_IMAGE} +# container_name: alertmanager +# restart: always +# volumes: +# - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml +# - ./config/email.tmpl:/etc/alertmanager/email.tmpl +# ports: +# - "19093:9093" +# networks: +# - openim +# +# grafana: +# image: ${GRAFANA_IMAGE} +# container_name: grafana +# user: root +# restart: always +# environment: +# - GF_SECURITY_ALLOW_EMBEDDING=true +# - GF_SESSION_COOKIE_SAMESITE=none +# - GF_SESSION_COOKIE_SECURE=true +# - GF_AUTH_ANONYMOUS_ENABLED=true +# - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin +# ports: +# - "13000:3000" +# volumes: +# - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana +# networks: +# - openim diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index 256214565b..a0345fc7a5 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -36,3 +36,26 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) { //export IMENV_OPENIM_RPC_USER_RPC_PORTS="10110,10111,10112" assert.Equal(t, []int{10110, 10111, 10112}, user.RPC.Ports) } + +func TestLoadNotificationConfig(t *testing.T) { + var noti Notification + err := LoadConfig("../../../config/notification.yml", "IMENV_NOTIFICATION", ¬i) + assert.Nil(t, err) + assert.Equal(t, "Your friend's profile has been changed", noti.FriendRemarkSet.OfflinePush.Title) +} + +func TestLoadOpenIMThirdConfig(t *testing.T) { + var third Third + err := LoadConfig("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", &third) + assert.Nil(t, err) + assert.Equal(t, "enabled", third.Object.Enable) + assert.Equal(t, "https://oss-cn-chengdu.aliyuncs.com", third.Object.Oss.Endpoint) + assert.Equal(t, "my_bucket_name", third.Object.Oss.Bucket) + assert.Equal(t, "https://my_bucket_name.oss-cn-chengdu.aliyuncs.com", third.Object.Oss.BucketURL) + assert.Equal(t, "AKID1234567890", third.Object.Oss.AccessKeyID) + assert.Equal(t, "abc123xyz789", third.Object.Oss.AccessKeySecret) + assert.Equal(t, "session_token_value", third.Object.Oss.SessionToken) // Uncomment if session token is needed + assert.Equal(t, true, third.Object.Oss.PublicRead) + + // Environment: IMENV_OPENIM_RPC_THIRD_OBJECT_ENABLE=enabled;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ENDPOINT=https://oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKET=my_bucket_name;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKETURL=https://my_bucket_name.oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYID=AKID1234567890;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYSECRET=abc123xyz789;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_SESSIONTOKEN=session_token_value;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_PUBLICREAD=true +} From e70695077a55eb0e2bb78ead088de82fb19f33fa Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 21 Aug 2024 15:23:43 +0800 Subject: [PATCH 66/91] Fix: solve conversation blocking in private chat when non friendship. (#2542) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: update contribute cla. * fix: solve conversations bug. --- .github/workflows/cla-assistant.yml | 6 ++-- CONTRIBUTING-zh_CN.md | 2 +- CONTRIBUTING.md | 2 +- internal/rpc/conversation/conversaion.go | 44 +++++++++++++++--------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cla-assistant.yml b/.github/workflows/cla-assistant.yml index 71bdb67992..7d44b05eb4 100644 --- a/.github/workflows/cla-assistant.yml +++ b/.github/workflows/cla-assistant.yml @@ -33,8 +33,8 @@ jobs: remote-repository-name: cla create-file-commit-message: 'Creating file for storing CLA Signatures' # signed-commit-message: '$contributorName has signed the CLA in $owner/$repo#$pullRequestNo' - custom-notsigned-prcomment: '💕 Thank you for your contribution and please kindly read and sign our [CLA Docs](https://github.com/OpenIM-Robot/cla/blob/main/README.md)' - custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' - custom-allsigned-prcomment: '🤖 All Contributors have signed the [CLA](https://github.com/OpenIM-Robot/cla/blob/main/README.md).
The signed information is recorded [🤖here](https://github.com/openim-sigs/cla/tree/main/signatures/cla.json)' + custom-notsigned-prcomment: '💕 Thank you for your contribution and please kindly read and sign our CLA. [CLA Docs](https://github.com/OpenIM-Robot/cla/blob/main/README.md)' + custom-pr-sign-comment: 'I have read the CLA Document and I hereby sign the CLA' + custom-allsigned-prcomment: '🤖 All Contributors have signed the [CLA](https://github.com/OpenIM-Robot/cla/blob/main/README.md).
The signed information is recorded [**here**](https://github.com/OpenIM-Robot/cla/blob/main/signatures/cla.json)' #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) #use-dco-flag: true - If you are using DCO instead of CLA diff --git a/CONTRIBUTING-zh_CN.md b/CONTRIBUTING-zh_CN.md index 2f875b1405..47965a9f49 100644 --- a/CONTRIBUTING-zh_CN.md +++ b/CONTRIBUTING-zh_CN.md @@ -77,7 +77,7 @@ git push origin fix-bug-123 ### 9. 签署 CLA 如果这是你第一次提交 PR，你需要在 PR 的评论中回复： ``` -The signature to be committed in order to sign the CLA +I have read the CLA Document and I hereby sign the CLA ``` ### 编程规范 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ed419b2f1..0aa07393ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,7 +75,7 @@ Go to your fork on GitHub and click the "Pull Request" button. Ensure the PR des ### 9. Sign the CLA If this is your first time submitting a PR, you will need to reply in the comments of the PR: ``` -The signature to be committed in order to sign the CLA +I have read the CLA Document and I hereby sign the CLA ``` ### Programming Standards diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 117ed8ca71..2fce2698ab 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -247,6 +247,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } conv = *cs[0] } + var conversation dbModel.Conversation conversation.ConversationID = req.Conversation.ConversationID conversation.ConversationType = req.Conversation.ConversationType @@ -259,12 +260,14 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver unequal++ } } + if req.Conversation.AttachedInfo != nil { m["attached_info"] = req.Conversation.AttachedInfo.Value if req.Conversation.AttachedInfo.Value != conv.AttachedInfo { unequal++ } } + if req.Conversation.Ex != nil { m["ex"] = req.Conversation.Ex.Value if req.Conversation.Ex.Value != conv.Ex { @@ -277,24 +280,48 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver unequal++ } } + if req.Conversation.GroupAtType != nil { m["group_at_type"] = req.Conversation.GroupAtType.Value if req.Conversation.GroupAtType.Value != conv.GroupAtType { unequal++ } } + if req.Conversation.MsgDestructTime != nil { m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value if req.Conversation.MsgDestructTime.Value != conv.MsgDestructTime { unequal++ } } + if req.Conversation.IsMsgDestruct != nil { m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value if req.Conversation.IsMsgDestruct.Value != conv.IsMsgDestruct { unequal++ } } + + if req.Conversation.BurnDuration != nil { + m["burn_duration"] = req.Conversation.BurnDuration.Value + if req.Conversation.BurnDuration.Value != conv.BurnDuration { + unequal++ + } + } + + if len(m) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, req.UserIDs, &conversation, m); err != nil { + return nil, err + } + } + + if unequal > 0 { + for _, v := range req.UserIDs { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + return &pbconversation.SetConversationsResp{}, nil + } + if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { @@ -313,23 +340,6 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } } - if req.Conversation.BurnDuration != nil { - m["burn_duration"] = req.Conversation.BurnDuration.Value - if req.Conversation.BurnDuration.Value != conv.BurnDuration { - unequal++ - } - } - - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, req.UserIDs, &conversation, m); err != nil { - return nil, err - } - - if unequal > 0 { - for _, v := range req.UserIDs { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) - } - } - return &pbconversation.SetConversationsResp{}, nil } From 97f4b51874c19d9e72509e82ceb0b8933c5af047 Mon Sep 17 00:00:00 2001 From: qinguoyi <1532979219@qq.com> Date: Wed, 21 Aug 2024 15:25:19 +0800 Subject: [PATCH 67/91] fix:get msg error (#2494) --- pkg/rpcclient/msg.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 124cc49af3..da556224fb 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -174,6 +174,9 @@ func (m *MessageRpcClient) GetMaxSeqs(ctx context.Context, conversationIDs []str resp, err := m.Client.GetMaxSeqs(ctx, &msg.GetMaxSeqsReq{ ConversationIDs: conversationIDs, }) + if err != nil { + return nil, err + } return resp.MaxSeqs, err } @@ -182,6 +185,9 @@ func (m *MessageRpcClient) GetHasReadSeqs(ctx context.Context, userID string, co UserID: userID, ConversationIDs: conversationIDs, }) + if err != nil { + return nil, err + } return resp.MaxSeqs, err } @@ -190,6 +196,9 @@ func (m *MessageRpcClient) GetMsgByConversationIDs(ctx context.Context, docIDs [ ConversationIDs: docIDs, MaxSeqs: seqs, }) + if err != nil { + return nil, err + } return resp.MsgDatas, err } From f44a97b8d285198da375aae200740d803184f647 Mon Sep 17 00:00:00 2001 From: qinguoyi <1532979219@qq.com> Date: Wed, 21 Aug 2024 15:26:53 +0800 Subject: [PATCH 68/91] fix:mgo delete err (#2496) --- pkg/common/storage/database/mgo/user.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 8978e64ebf..ee92b75544 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -167,6 +167,10 @@ func (u *UserMgo) DeleteUserCommand(ctx context.Context, userID string, Type int filter := bson.M{"userID": userID, "type": Type, "uuid": UUID} result, err := collection.DeleteOne(ctx, filter) + // when err is not nil, result might be nil + if err != nil { + return errs.Wrap(err) + } if result.DeletedCount == 0 { // No records found to update return errs.Wrap(errs.ErrRecordNotFound) From a559bfe32367f420ecb47a9a8a6b85f8a8071051 Mon Sep 17 00:00:00 2001 From: qinguoyi <1532979219@qq.com> Date: Wed, 21 Aug 2024 15:27:51 +0800 Subject: [PATCH 69/91] fix:doPut error (#2495) --- tools/url2im/pkg/manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/url2im/pkg/manage.go b/tools/url2im/pkg/manage.go index 5e1626da9f..3664baa25b 100644 --- a/tools/url2im/pkg/manage.go +++ b/tools/url2im/pkg/manage.go @@ -234,7 +234,7 @@ func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) { } for i, currentPartSize := range part.PartSizes { md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize)) - if m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil { + if err := m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil { return "", err } if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] { From d938754ff60feb83badff27589f949ad7f12dde3 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 22 Aug 2024 17:43:13 +0800 Subject: [PATCH 70/91] feat: update set conversation logic. (#2544) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * feat: update set conversation logic. * content format. * update setConversation logic. * update conversation logic. * update conversation logic. * update logic. * update logic. --- internal/rpc/conversation/conversaion.go | 154 ++++++++++++++--------- 1 file changed, 94 insertions(+), 60 deletions(-) diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 2fce2698ab..f00c970e65 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -221,11 +221,11 @@ func (c *conversationServer) SetConversation(ctx context.Context, req *pbconvers return resp, nil } -// nolint func (c *conversationServer) SetConversations(ctx context.Context, req *pbconversation.SetConversationsReq) (*pbconversation.SetConversationsResp, error) { if req.Conversation == nil { return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") } + if req.Conversation.ConversationType == constant.WriteGroupChatType { groupInfo, err := c.groupRpcClient.GetGroupInfo(ctx, req.Conversation.GroupID) if err != nil { @@ -235,17 +235,20 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed") } } - var unequal int - var conv dbModel.Conversation - if len(req.UserIDs) == 1 { - cs, err := c.conversationDatabase.FindConversations(ctx, req.UserIDs[0], []string{req.Conversation.ConversationID}) + + conversationMap := make(map[string]*dbModel.Conversation) + var needUpdateUsersList []string + + for _, userID := range req.UserIDs { + conversationList, err := c.conversationDatabase.FindConversations(ctx, userID, []string{req.Conversation.ConversationID}) if err != nil { return nil, err } - if len(cs) == 0 { - return nil, errs.ErrRecordNotFound.WrapMsg("conversation not found") + if len(conversationList) != 0 { + conversationMap[userID] = conversationList[0] + } else { + needUpdateUsersList = append(needUpdateUsersList, userID) } - conv = *cs[0] } var conversation dbModel.Conversation @@ -253,91 +256,122 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver conversation.ConversationType = req.Conversation.ConversationType conversation.UserID = req.Conversation.UserID conversation.GroupID = req.Conversation.GroupID + m := make(map[string]any) - if req.Conversation.RecvMsgOpt != nil { - m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value - if req.Conversation.RecvMsgOpt.Value != conv.RecvMsgOpt { - unequal++ - } - } - if req.Conversation.AttachedInfo != nil { - m["attached_info"] = req.Conversation.AttachedInfo.Value - if req.Conversation.AttachedInfo.Value != conv.AttachedInfo { - unequal++ + setConversationFieldsFunc := func() { + if req.Conversation.RecvMsgOpt != nil { + m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value + } + if req.Conversation.AttachedInfo != nil { + m["attached_info"] = req.Conversation.AttachedInfo.Value + } + if req.Conversation.Ex != nil { + m["ex"] = req.Conversation.Ex.Value + } + if req.Conversation.IsPinned != nil { + m["is_pinned"] = req.Conversation.IsPinned.Value + } + if req.Conversation.GroupAtType != nil { + m["group_at_type"] = req.Conversation.GroupAtType.Value + } + if req.Conversation.MsgDestructTime != nil { + m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + } + if req.Conversation.MsgDestructTime != nil { + m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + } + if req.Conversation.BurnDuration != nil { + m["burn_duration"] = req.Conversation.BurnDuration.Value } } - if req.Conversation.Ex != nil { - m["ex"] = req.Conversation.Ex.Value - if req.Conversation.Ex.Value != conv.Ex { - unequal++ + // set need set field in conversation + setConversationFieldsFunc() + + for userID := range conversationMap { + unequal := len(m) + + if req.Conversation.RecvMsgOpt != nil { + if req.Conversation.RecvMsgOpt.Value != conversationMap[userID].RecvMsgOpt { + unequal-- + } } - } - if req.Conversation.IsPinned != nil { - m["is_pinned"] = req.Conversation.IsPinned.Value - if req.Conversation.IsPinned.Value != conv.IsPinned { - unequal++ + + if req.Conversation.AttachedInfo != nil { + if req.Conversation.AttachedInfo.Value != conversationMap[userID].AttachedInfo { + unequal-- + } } - } - if req.Conversation.GroupAtType != nil { - m["group_at_type"] = req.Conversation.GroupAtType.Value - if req.Conversation.GroupAtType.Value != conv.GroupAtType { - unequal++ + if req.Conversation.Ex != nil { + if req.Conversation.Ex.Value != conversationMap[userID].Ex { + unequal-- + } + } + if req.Conversation.IsPinned != nil { + m["is_pinned"] = req.Conversation.IsPinned.Value + if req.Conversation.IsPinned.Value != conversationMap[userID].IsPinned { + unequal-- + } } - } - if req.Conversation.MsgDestructTime != nil { - m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value - if req.Conversation.MsgDestructTime.Value != conv.MsgDestructTime { - unequal++ + if req.Conversation.GroupAtType != nil { + if req.Conversation.GroupAtType.Value != conversationMap[userID].GroupAtType { + unequal-- + } } - } - if req.Conversation.IsMsgDestruct != nil { - m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value - if req.Conversation.IsMsgDestruct.Value != conv.IsMsgDestruct { - unequal++ + if req.Conversation.MsgDestructTime != nil { + if req.Conversation.MsgDestructTime.Value != conversationMap[userID].MsgDestructTime { + unequal-- + } } - } - if req.Conversation.BurnDuration != nil { - m["burn_duration"] = req.Conversation.BurnDuration.Value - if req.Conversation.BurnDuration.Value != conv.BurnDuration { - unequal++ + if req.Conversation.IsMsgDestruct != nil { + if req.Conversation.IsMsgDestruct.Value != conversationMap[userID].IsMsgDestruct { + unequal-- + } } - } - if len(m) != 0 { - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, req.UserIDs, &conversation, m); err != nil { - return nil, err + if req.Conversation.BurnDuration != nil { + if req.Conversation.BurnDuration.Value != conversationMap[userID].BurnDuration { + unequal-- + } } - } - if unequal > 0 { - for _, v := range req.UserIDs { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + if unequal > 0 { + needUpdateUsersList = append(needUpdateUsersList, userID) } - return &pbconversation.SetConversationsResp{}, nil } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { - conversation2 := conversation - conversation2.OwnerUserID = ownerUserID - conversation2.IsPrivateChat = req.Conversation.IsPrivateChat.Value - conversations = append(conversations, &conversation2) + transConversation := conversation + transConversation.OwnerUserID = ownerUserID + transConversation.IsPrivateChat = req.Conversation.IsPrivateChat.Value + conversations = append(conversations, &transConversation) } if err := c.conversationDatabase.SyncPeerUserPrivateConversationTx(ctx, conversations); err != nil { return nil, err } + for _, userID := range req.UserIDs { c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } + } else { + if len(m) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } + + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + } } return &pbconversation.SetConversationsResp{}, nil From 7deb45e35d61d16c006f02c0baa4cb1f16077706 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:34:58 +0800 Subject: [PATCH 71/91] Groupmsg (#2548) * feat: add EnableHistoryForNewMembers * feat: change name * refactor: move first create conversation * fix: set min seq --- config/openim-rpc-group.yml | 3 ++ go.mod | 5 +- go.sum | 6 +-- internal/rpc/conversation/conversaion.go | 8 ++++ internal/rpc/group/group.go | 35 ++++++++------ internal/rpc/group/notification.go | 61 +++++++++++++----------- pkg/common/config/config.go | 3 +- pkg/rpcclient/conversation.go | 5 ++ 8 files changed, 76 insertions(+), 50 deletions(-) diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index f49c46207f..5fff8b9fe3 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -11,3 +11,6 @@ prometheus: enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup ports: [ 20103 ] + + +enableHistoryForNewMembers: true \ No newline at end of file diff --git a/go.mod b/go.mod index fba1499fe1..1d7c648f7e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69 + github.com/openimsdk/protocol v0.0.72-alpha.2 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -41,6 +41,7 @@ require ( github.com/spf13/viper v1.18.2 github.com/stathat/consistent v1.0.0 go.uber.org/automaxprocs v1.5.3 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.6.0 ) @@ -74,7 +75,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chai2010/webp v1.1.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/coreos/go-semver v0.3.0 // indirect @@ -170,7 +170,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect diff --git a/go.sum b/go.sum index 1a8e1d76d8..5e0874723d 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,6 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk= -github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -321,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69 h1:dVi8meSg8kmUzSH1XQab4MjihqKkkcCAmt1BYXPJuXo= -github.com/openimsdk/protocol v0.0.69/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.2 h1:H5IcoAR4jTzJ7uvmFmc6/6rPOKEGRLWnu+qre8B5hk0= +github.com/openimsdk/protocol v0.0.72-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index f00c970e65..cb6546f9b0 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -436,6 +436,14 @@ func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbc return &pbconversation.SetConversationMaxSeqResp{}, nil } +func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, + map[string]any{"min_seq": req.MinSeq}); err != nil { + return nil, err + } + return &pbconversation.SetConversationMinSeqResp{}, nil +} + func (c *conversationServer) GetConversationIDs(ctx context.Context, req *pbconversation.GetConversationIDsReq) (*pbconversation.GetConversationIDsResp, error) { conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) if err != nil { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index aa12c9d0f1..bb3bfcaeea 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -105,13 +105,20 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database gs.user = userRpcClient - gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { - users, err := userRpcClient.GetUsersInfo(ctx, userIDs) - if err != nil { - return nil, err - } - return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil - }) + gs.notification = NewGroupNotificationSender( + database, + &msgRpcClient, + &userRpcClient, + &conversationRpcClient, + config, + func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { + users, err := userRpcClient.GetUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil + }, + ) localcache.InitLocalCache(&config.LocalCacheConfig) gs.conversationRpcClient = conversationRpcClient gs.msgRpcClient = msgRpcClient @@ -450,10 +457,10 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite if err := s.db.CreateGroup(ctx, nil, groupMembers); err != nil { return nil, err } - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, req.InvitedUserIDs); err != nil { + + if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.InvitedUserIDs...); err != nil { return nil, err } - s.notification.MemberInvitedNotification(ctx, req.GroupID, req.Reason, req.InvitedUserIDs) return &pbgroup.InviteUserToGroupResp{}, nil } @@ -822,14 +829,13 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup } switch req.HandleResult { case constant.GroupResponseAgree: - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, []string{req.FromUserID}); err != nil { - return nil, err - } s.notification.GroupApplicationAcceptedNotification(ctx, req) if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - s.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID) + if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { + return nil, err + } } case constant.GroupResponseRefuse: s.notification.GroupApplicationRejectedNotification(ctx, req) @@ -889,10 +895,9 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) return nil, err } - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, []string{req.InviterUserID}); err != nil { + if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID); err != nil { return nil, err } - s.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID) s.webhookAfterJoinGroup(ctx, &s.config.WebhooksConfig.AfterJoinGroup, req) return &pbgroup.JoinGroupResp{}, nil diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 9815167e9b..72ca9b8130 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -21,6 +21,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -43,12 +44,22 @@ const ( adminReceiver ) -func NewGroupNotificationSender(db controller.GroupDatabase, msgRpcClient *rpcclient.MessageRpcClient, userRpcClient *rpcclient.UserRpcClient, config *Config, fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error)) *GroupNotificationSender { +func NewGroupNotificationSender( + db controller.GroupDatabase, + msgRpcClient *rpcclient.MessageRpcClient, + userRpcClient *rpcclient.UserRpcClient, + conversationRpcClient *rpcclient.ConversationRpcClient, + config *Config, + fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error), +) *GroupNotificationSender { return &GroupNotificationSender{ NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(msgRpcClient), rpcclient.WithUserRpcClient(userRpcClient)), getUsersInfo: fn, db: db, config: config, + + conversationRpcClient: conversationRpcClient, + msgRpcClient: msgRpcClient, } } @@ -57,6 +68,9 @@ type GroupNotificationSender struct { getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) db controller.GroupDatabase config *Config + + conversationRpcClient *rpcclient.ConversationRpcClient + msgRpcClient *rpcclient.MessageRpcClient } func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, members ...*model.GroupMember) error { @@ -494,50 +508,43 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } -func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, groupID, reason string, invitedUserIDList []string) { +func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { var err error defer func() { if err != nil { log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) } }() - var group *sdkws.GroupInfo - group, err = g.getGroupInfo(ctx, groupID) - if err != nil { - return + + if !g.config.RpcConfig.EnableHistoryForNewMembers { + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) + maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + if err != nil { + return err + } + err = g.conversationRpcClient.SetConversationMinSeq(ctx, entrantUserID, conversationID, maxSeq) + if err != nil { + return err + } } - var users []*sdkws.GroupMemberFullInfo - users, err = g.getGroupMembers(ctx, groupID, invitedUserIDList) - if err != nil { - return + if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, entrantUserID); err != nil { + return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} - err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) - g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) -} -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID string) { - var err error - defer func() { - if err != nil { - log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) - } - }() var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, groupID) if err != nil { - return + return err } - var user *sdkws.GroupMemberFullInfo - user, err = g.getGroupMember(ctx, groupID, entrantUserID) + users, err := g.getGroupMembers(ctx, groupID, entrantUserID) if err != nil { - return + return err } - tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} + tips := &sdkws.MemberEnterTips{Group: group, EntrantUsers: users} g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) + return nil } func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c6c672eb8d..fc6d3001ff 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -258,7 +258,8 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + EnableHistoryForNewMembers bool `mapstructure:"enableHistoryForNewMembers"` } type Msg struct { diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go index 8f95f86a6c..ccca856194 100644 --- a/pkg/rpcclient/conversation.go +++ b/pkg/rpcclient/conversation.go @@ -77,6 +77,11 @@ func (c *ConversationRpcClient) SetConversationMaxSeq(ctx context.Context, owner return err } +func (c *ConversationRpcClient) SetConversationMinSeq(ctx context.Context, ownerUserIDs []string, conversationID string, minSeq int64) error { + _, err := c.Client.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{OwnerUserID: ownerUserIDs, ConversationID: conversationID, MinSeq: minSeq}) + return err +} + func (c *ConversationRpcClient) SetConversations(ctx context.Context, userIDs []string, conversation *pbconversation.ConversationReq) error { _, err := c.Client.SetConversations(ctx, &pbconversation.SetConversationsReq{UserIDs: userIDs, Conversation: conversation}) return err From fa1467878beb051daa79b3d98a60978af1afbc20 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 26 Aug 2024 14:29:10 +0800 Subject: [PATCH 72/91] feat: implement `SetGroupInfoEX` interface. (#2552) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * feat: implement `SetGroupInfoEX` interface. * remove null interface. * update router. --- config/webhooks.yml | 7 + go.mod | 2 +- go.sum | 4 +- internal/api/group.go | 4 + internal/api/router.go | 7 +- internal/api/user.go | 2 + internal/rpc/group/callback.go | 71 +++++ internal/rpc/group/db_map.go | 33 ++ internal/rpc/group/group.go | 560 ++++++++++++++++++++------------- internal/rpc/user/user.go | 28 +- pkg/callbackstruct/constant.go | 2 + pkg/callbackstruct/group.go | 46 +++ pkg/common/config/config.go | 2 + 13 files changed, 522 insertions(+), 246 deletions(-) diff --git a/config/webhooks.yml b/config/webhooks.yml index 421522ff86..24fb2413a0 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -130,6 +130,13 @@ beforeSetGroupInfo: enable: false timeout: 5 failedContinue: true +afterSetGroupInfoEX: + enable: false + timeout: 5 +beforeSetGroupInfoEX: + enable: false + timeout: 5 + failedContinue: true afterRevokeMsg: enable: false timeout: 5 diff --git a/go.mod b/go.mod index 1d7c648f7e..2d3534b561 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.2 + github.com/openimsdk/protocol v0.0.72-alpha.5 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5e0874723d..30fdacb4b9 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.2 h1:H5IcoAR4jTzJ7uvmFmc6/6rPOKEGRLWnu+qre8B5hk0= -github.com/openimsdk/protocol v0.0.72-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.5 h1:1Xjyx6ivTb782Sm7wMJXCLZP80iXADVo1CySis1rOG0= +github.com/openimsdk/protocol v0.0.72-alpha.5/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/group.go b/internal/api/group.go index bff0089748..14f50cacdd 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -35,6 +35,10 @@ func (o *GroupApi) SetGroupInfo(c *gin.Context) { a2r.Call(group.GroupClient.SetGroupInfo, o.Client, c) } +func (o *GroupApi) SetGroupInfoEX(c *gin.Context) { + a2r.Call(group.GroupClient.SetGroupInfoEX, o.Client, c) +} + func (o *GroupApi) JoinGroup(c *gin.Context) { a2r.Call(group.GroupClient.JoinGroup, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 0667c3e751..1718505817 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,12 +2,16 @@ package api import ( "fmt" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "net/http" + "strings" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" @@ -16,8 +20,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "net/http" - "strings" ) func prommetricsGin() gin.HandlerFunc { @@ -112,6 +114,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En { groupRouterGroup.POST("/create_group", g.CreateGroup) groupRouterGroup.POST("/set_group_info", g.SetGroupInfo) + groupRouterGroup.POST("/set_group_info_ex", g.SetGroupInfoEX) groupRouterGroup.POST("/join_group", g.JoinGroup) groupRouterGroup.POST("/quit_group", g.QuitGroup) groupRouterGroup.POST("/group_application_response", g.ApplicationGroupResponse) diff --git a/internal/api/user.go b/internal/api/user.go index d48111b9eb..dba7cd3129 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -36,9 +36,11 @@ func (u *UserApi) UserRegister(c *gin.Context) { a2r.Call(user.UserClient.UserRegister, u.Client, c) } +// UpdateUserInfo is deprecated. Use UpdateUserInfoEx func (u *UserApi) UpdateUserInfo(c *gin.Context) { a2r.Call(user.UserClient.UpdateUserInfo, u.Client, c) } + func (u *UserApi) UpdateUserInfoEx(c *gin.Context) { a2r.Call(user.UserClient.UpdateUserInfoEx, u.Client, c) } diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index f877aa64a8..5e3dc9b9c3 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -358,3 +358,74 @@ func (s *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *confi } s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoResp{}, after) } + +func (s *groupServer) webhookBeforeSetGroupInfoEX(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoEXReq) error { + return webhook.WithCondition(ctx, before, func(ctx context.Context) error { + cbReq := &callbackstruct.CallbackBeforeSetGroupInfoEXReq{ + CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoCommand, + GroupID: req.GroupInfoForSetEX.GroupID, + GroupName: req.GroupInfoForSetEX.GroupName, + Notification: req.GroupInfoForSetEX.Notification, + Introduction: req.GroupInfoForSetEX.Introduction, + FaceURL: req.GroupInfoForSetEX.FaceURL, + } + + if req.GroupInfoForSetEX.Ex != nil { + cbReq.Ex = req.GroupInfoForSetEX.Ex + } + log.ZDebug(ctx, "debug CallbackBeforeSetGroupInfoEX", "ex", cbReq.Ex) + + if req.GroupInfoForSetEX.NeedVerification != nil { + cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + } + if req.GroupInfoForSetEX.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + } + if req.GroupInfoForSetEX.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + } + + resp := &callbackstruct.CallbackBeforeSetGroupInfoEXResp{} + + if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + return err + } + + datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupID, &resp.GroupID) + datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupName, &resp.GroupName) + datautil.NotNilReplace(&req.GroupInfoForSetEX.FaceURL, &resp.FaceURL) + datautil.NotNilReplace(&req.GroupInfoForSetEX.Introduction, &resp.Introduction) + datautil.NotNilReplace(&req.GroupInfoForSetEX.Ex, &resp.Ex) + datautil.NotNilReplace(&req.GroupInfoForSetEX.NeedVerification, &resp.NeedVerification) + datautil.NotNilReplace(&req.GroupInfoForSetEX.LookMemberInfo, &resp.LookMemberInfo) + datautil.NotNilReplace(&req.GroupInfoForSetEX.ApplyMemberFriend, &resp.ApplyMemberFriend) + + return nil + }) +} + +func (s *groupServer) webhookAfterSetGroupInfoEX(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoEXReq) { + cbReq := &callbackstruct.CallbackAfterSetGroupInfoEXReq{ + CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoCommand, + GroupID: req.GroupInfoForSetEX.GroupID, + GroupName: req.GroupInfoForSetEX.GroupName, + Notification: req.GroupInfoForSetEX.Notification, + Introduction: req.GroupInfoForSetEX.Introduction, + FaceURL: req.GroupInfoForSetEX.FaceURL, + } + + if req.GroupInfoForSetEX.Ex != nil { + cbReq.Ex = req.GroupInfoForSetEX.Ex + } + if req.GroupInfoForSetEX.NeedVerification != nil { + cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + } + if req.GroupInfoForSetEX.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + } + if req.GroupInfoForSetEX.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + } + + s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoEXResp{}, after) +} diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index b4b503b950..08895f9c5f 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -54,6 +54,39 @@ func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[s return m } +func UpdateGroupInfoEXMap(ctx context.Context, group *sdkws.GroupInfoForSetEX) map[string]any { + m := make(map[string]any) + + if group.GroupName != "" { + m["group_name"] = group.GroupName + } + if group.Notification != nil { + m["notification"] = group.Notification.Value + m["notification_update_time"] = time.Now() + m["notification_user_id"] = mcontext.GetOpUserID(ctx) + } + if group.Introduction != nil { + m["introduction"] = group.Introduction.Value + } + if group.FaceURL != nil { + m["face_url"] = group.FaceURL.Value + } + if group.NeedVerification != nil { + m["need_verification"] = group.NeedVerification.Value + } + if group.LookMemberInfo != nil { + m["look_member_info"] = group.LookMemberInfo.Value + } + if group.ApplyMemberFriend != nil { + m["apply_member_friend"] = group.ApplyMemberFriend.Value + } + if group.Ex != nil { + m["ex"] = group.Ex.Value + } + + return m +} + func UpdateGroupStatusMap(status int) map[string]any { return map[string]any{ "status": status, diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index bb3bfcaeea..9589ed2494 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -128,8 +128,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return nil } -func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgroup.NotificationUserInfoUpdateReq) (*pbgroup.NotificationUserInfoUpdateResp, error) { - members, err := s.db.FindGroupMemberUser(ctx, nil, req.UserID) +func (g *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgroup.NotificationUserInfoUpdateReq) (*pbgroup.NotificationUserInfoUpdateResp, error) { + members, err := g.db.FindGroupMemberUser(ctx, nil, req.UserID) if err != nil { return nil, err } @@ -141,22 +141,22 @@ func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro groupIDs = append(groupIDs, member.GroupID) } for _, groupID := range groupIDs { - if err := s.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + if err := g.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { return nil, err } } for _, groupID := range groupIDs { - s.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) + g.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) } - if err = s.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { + if err = g.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { return nil, err } return &pbgroup.NotificationUserInfoUpdateResp{}, nil } -func (s *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - groupMember, err := s.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) +func (g *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + groupMember, err := g.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) if err != nil { return err } @@ -167,11 +167,11 @@ func (s *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error return nil } -func (s *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string, complete bool) (map[string]*sdkws.PublicUserInfo, error) { +func (g *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string, complete bool) (map[string]*sdkws.PublicUserInfo, error) { if len(userIDs) == 0 { return map[string]*sdkws.PublicUserInfo{}, nil } - users, err := s.user.GetPublicUserInfos(ctx, userIDs, complete) + users, err := g.user.GetPublicUserInfos(ctx, userIDs, complete) if err != nil { return nil, err } @@ -180,16 +180,16 @@ func (s *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string }), nil } -func (s *groupServer) IsNotFound(err error) bool { +func (g *groupServer) IsNotFound(err error) bool { return errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) } -func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { +func (g *groupServer) GenGroupID(ctx context.Context, groupID *string) error { if *groupID != "" { - _, err := s.db.TakeGroup(ctx, *groupID) + _, err := g.db.TakeGroup(ctx, *groupID) if err == nil { return servererrs.ErrGroupIDExisted.WrapMsg("group id existed " + *groupID) - } else if s.IsNotFound(err) { + } else if g.IsNotFound(err) { return nil } else { return err @@ -200,10 +200,10 @@ func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { bi := big.NewInt(0) bi.SetString(id[0:8], 16) id = bi.String() - _, err := s.db.TakeGroup(ctx, id) + _, err := g.db.TakeGroup(ctx, id) if err == nil { continue - } else if s.IsNotFound(err) { + } else if g.IsNotFound(err) { *groupID = id return nil } else { @@ -213,14 +213,14 @@ func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { return servererrs.ErrData.WrapMsg("group id gen error") } -func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupReq) (*pbgroup.CreateGroupResp, error) { +func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupReq) (*pbgroup.CreateGroupResp, error) { if req.GroupInfo.GroupType != constant.WorkingGroup { return nil, errs.ErrArgs.WrapMsg(fmt.Sprintf("group type only supports %d", constant.WorkingGroup)) } if req.OwnerUserID == "" { return nil, errs.ErrArgs.WrapMsg("no group owner") } - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -234,7 +234,7 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, errs.ErrArgs.WrapMsg("group member repeated") } - userMap, err := s.user.GetUsersInfoMap(ctx, userIDs) + userMap, err := g.user.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -243,13 +243,13 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, servererrs.ErrUserIDNotFound.WrapMsg("user not found") } - if err := s.webhookBeforeCreateGroup(ctx, &s.config.WebhooksConfig.BeforeCreateGroup, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeCreateGroup(ctx, &g.config.WebhooksConfig.BeforeCreateGroup, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } var groupMembers []*model.GroupMember group := convert.Pb2DBGroupInfo(req.GroupInfo) - if err := s.GenGroupID(ctx, &group.GroupID); err != nil { + if err := g.GenGroupID(ctx, &group.GroupID); err != nil { return nil, err } @@ -278,11 +278,11 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR joinGroupFunc(userID, constant.GroupOrdinaryUsers) } - if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if err := s.db.CreateGroup(ctx, []*model.Group{group}, groupMembers); err != nil { + if err := g.db.CreateGroup(ctx, []*model.Group{group}, groupMembers); err != nil { return nil, err } resp := &pbgroup.CreateGroupResp{GroupInfo: &sdkws.GroupInfo{}} @@ -292,17 +292,17 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR tips := &sdkws.GroupCreatedTips{ Group: resp.GroupInfo, OperationTime: group.CreateTime.UnixMilli(), - GroupOwnerUser: s.groupMemberDB2PB(groupMembers[0], userMap[groupMembers[0].UserID].AppMangerLevel), + GroupOwnerUser: g.groupMemberDB2PB(groupMembers[0], userMap[groupMembers[0].UserID].AppMangerLevel), } for _, member := range groupMembers { member.Nickname = userMap[member.UserID].Nickname - tips.MemberList = append(tips.MemberList, s.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel)) + tips.MemberList = append(tips.MemberList, g.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel)) if member.UserID == opUserID { - tips.OpUser = s.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel) + tips.OpUser = g.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel) break } } - s.notification.GroupCreatedNotification(ctx, tips) + g.notification.GroupCreatedNotification(ctx, tips) reqCallBackAfter := &pbgroup.CreateGroupReq{ MemberUserIDs: userIDs, @@ -311,16 +311,16 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR AdminUserIDs: req.AdminUserIDs, } - s.webhookAfterCreateGroup(ctx, &s.config.WebhooksConfig.AfterCreateGroup, reqCallBackAfter) + g.webhookAfterCreateGroup(ctx, &g.config.WebhooksConfig.AfterCreateGroup, reqCallBackAfter) return resp, nil } -func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJoinedGroupListReq) (*pbgroup.GetJoinedGroupListResp, error) { - if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { +func (g *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJoinedGroupListReq) (*pbgroup.GetJoinedGroupListResp, error) { + if err := authverify.CheckAccessV3(ctx, req.FromUserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } - total, members, err := s.db.PageGetJoinGroup(ctx, req.FromUserID, req.Pagination) + total, members, err := g.db.PageGetJoinGroup(ctx, req.FromUserID, req.Pagination) if err != nil { return nil, err } @@ -332,19 +332,19 @@ func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJo groupIDs := datautil.Slice(members, func(e *model.GroupMember) string { return e.GroupID }) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { @@ -362,14 +362,14 @@ func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJo return &resp, nil } -func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.InviteUserToGroupReq) (*pbgroup.InviteUserToGroupResp, error) { +func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.InviteUserToGroupReq) (*pbgroup.InviteUserToGroupResp, error) { if len(req.InvitedUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("user empty") } if datautil.Duplicate(req.InvitedUserIDs) { return nil, errs.ErrArgs.WrapMsg("userID duplicate") } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -378,7 +378,7 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed checking group status found it dismissed") } - userMap, err := s.user.GetUsersInfoMap(ctx, req.InvitedUserIDs) + userMap, err := g.user.GetUsersInfoMap(ctx, req.InvitedUserIDs) if err != nil { return nil, err } @@ -389,24 +389,24 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite var groupMember *model.GroupMember var opUserID string - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { opUserID = mcontext.GetOpUserID(ctx) var err error - groupMember, err = s.db.TakeGroupMember(ctx, req.GroupID, opUserID) + groupMember, err = g.db.TakeGroupMember(ctx, req.GroupID, opUserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, groupMember); err != nil { + if err := g.PopulateGroupMember(ctx, groupMember); err != nil { return nil, err } } - if err := s.webhookBeforeInviteUserToGroup(ctx, &s.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeInviteUserToGroup(ctx, &g.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } if group.NeedVerification == constant.AllNeedVerification { - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if !(groupMember.RoleLevel == constant.GroupOwner || groupMember.RoleLevel == constant.GroupAdmin) { var requests []*model.GroupRequest for _, userID := range req.InvitedUserIDs { @@ -419,11 +419,11 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite HandledTime: time.Unix(0, 0), }) } - if err := s.db.CreateGroupRequest(ctx, requests); err != nil { + if err := g.db.CreateGroupRequest(ctx, requests); err != nil { return nil, err } for _, request := range requests { - s.notification.JoinGroupApplicationNotification(ctx, &pbgroup.JoinGroupReq{ + g.notification.JoinGroupApplicationNotification(ctx, &pbgroup.JoinGroupReq{ GroupID: request.GroupID, ReqMessage: request.ReqMsg, JoinSource: request.JoinSource, @@ -450,26 +450,26 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite groupMembers = append(groupMembers, member) } - if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if err := s.db.CreateGroup(ctx, nil, groupMembers); err != nil { + if err := g.db.CreateGroup(ctx, nil, groupMembers); err != nil { return nil, err } - if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.InvitedUserIDs...); err != nil { + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.InvitedUserIDs...); err != nil { return nil, err } return &pbgroup.InviteUserToGroupResp{}, nil } -func (s *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGroupAllMemberReq) (*pbgroup.GetGroupAllMemberResp, error) { - members, err := s.db.FindGroupMemberAll(ctx, req.GroupID) +func (g *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGroupAllMemberReq) (*pbgroup.GetGroupAllMemberResp, error) { + members, err := g.db.FindGroupMemberAll(ctx, req.GroupID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } var resp pbgroup.GetGroupAllMemberResp @@ -479,21 +479,21 @@ func (s *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGro return &resp, nil } -func (s *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { +func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { var ( total int64 members []*model.GroupMember err error ) if req.Keyword == "" { - total, members, err = s.db.PageGetGroupMember(ctx, req.GroupID, req.Pagination) + total, members, err = g.db.PageGetGroupMember(ctx, req.GroupID, req.Pagination) } else { - members, err = s.db.FindGroupMemberAll(ctx, req.GroupID) + members, err = g.db.FindGroupMemberAll(ctx, req.GroupID) } if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } if req.Keyword != "" { @@ -523,8 +523,8 @@ func (s *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGr }, nil } -func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGroupMemberReq) (*pbgroup.KickGroupMemberResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGroupMemberReq) (*pbgroup.KickGroupMemberResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -538,7 +538,7 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou if datautil.Contain(opUserID, req.KickedUserIDs...) { return nil, errs.ErrArgs.WrapMsg("opUserID in KickedUserIDs") } - owner, err := s.db.TakeGroupOwner(ctx, req.GroupID) + owner, err := g.db.TakeGroupOwner(ctx, req.GroupID) if err != nil { return nil, err } @@ -546,18 +546,18 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou return nil, errs.ErrArgs.WrapMsg("ownerUID can not Kick") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, append(req.KickedUserIDs, opUserID)) + members, err := g.db.FindGroupMembers(ctx, req.GroupID, append(req.KickedUserIDs, opUserID)) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } memberMap := make(map[string]*model.GroupMember) for i, member := range members { memberMap[member.UserID] = members[i] } - isAppManagerUid := authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) opMember := memberMap[opUserID] for _, userID := range req.KickedUserIDs { member, ok := memberMap[userID] @@ -581,11 +581,11 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou } } } - num, err := s.db.FindGroupMemberNum(ctx, req.GroupID) + num, err := g.db.FindGroupMemberNum(ctx, req.GroupID) if err != nil { return nil, err } - ownerUserIDs, err := s.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupOwner) + ownerUserIDs, err := g.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupOwner) if err != nil { return nil, err } @@ -593,7 +593,7 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou if len(ownerUserIDs) > 0 { ownerUserID = ownerUserIDs[0] } - if err := s.db.DeleteGroupMember(ctx, group.GroupID, req.KickedUserIDs); err != nil { + if err := g.db.DeleteGroupMember(ctx, group.GroupID, req.KickedUserIDs); err != nil { return nil, err } tips := &sdkws.MemberKickedTips{ @@ -624,23 +624,23 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou for _, userID := range req.KickedUserIDs { tips.KickedUserList = append(tips.KickedUserList, convert.Db2PbGroupMember(memberMap[userID])) } - s.notification.MemberKickedNotification(ctx, tips) - if err := s.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { + g.notification.MemberKickedNotification(ctx, tips) + if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { return nil, err } - s.webhookAfterKickGroupMember(ctx, &s.config.WebhooksConfig.AfterKickGroupMember, req) + g.webhookAfterKickGroupMember(ctx, &g.config.WebhooksConfig.AfterKickGroupMember, req) return &pbgroup.KickGroupMemberResp{}, nil } -func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetGroupMembersInfoReq) (*pbgroup.GetGroupMembersInfoResp, error) { +func (g *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetGroupMembersInfoReq) (*pbgroup.GetGroupMembersInfoResp, error) { if len(req.UserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDs empty") } if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } - members, err := s.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) + members, err := g.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err } @@ -649,15 +649,15 @@ func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG }, nil } -func (s *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { +func (g *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { if len(userIDs) == 0 { return nil, nil } - members, err := s.db.FindGroupMembers(ctx, groupID, userIDs) + members, err := g.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } return datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { @@ -666,8 +666,8 @@ func (s *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, u } // GetGroupApplicationList handles functions that get a list of group requests. -func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { - groupIDs, err := s.db.FindUserManagedGroupID(ctx, req.FromUserID) +func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { + groupIDs, err := g.db.FindUserManagedGroupID(ctx, req.FromUserID) if err != nil { return nil, err } @@ -675,7 +675,7 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. if len(groupIDs) == 0 { return resp, nil } - total, groupRequests, err := s.db.PageGroupRequest(ctx, groupIDs, req.Pagination) + total, groupRequests, err := g.db.PageGroupRequest(ctx, groupIDs, req.Pagination) if err != nil { return nil, err } @@ -689,11 +689,11 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. userIDs = append(userIDs, gr.UserID) } userIDs = datautil.Distinct(userIDs) - userMap, err := s.user.GetPublicUserInfoMap(ctx, userIDs, true) + userMap, err := g.user.GetPublicUserInfoMap(ctx, userIDs, true) if err != nil { return nil, err } - groups, err := s.db.FindGroup(ctx, datautil.Distinct(groupIDs)) + groups, err := g.db.FindGroup(ctx, datautil.Distinct(groupIDs)) if err != nil { return nil, err } @@ -703,15 +703,15 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. if ids := datautil.Single(datautil.Keys(groupMap), groupIDs); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { @@ -727,11 +727,11 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. return resp, nil } -func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsInfoReq) (*pbgroup.GetGroupsInfoResp, error) { +func (g *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsInfoReq) (*pbgroup.GetGroupsInfoResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupID is empty") } - groups, err := s.getGroupsInfo(ctx, req.GroupIDs) + groups, err := g.getGroupsInfo(ctx, req.GroupIDs) if err != nil { return nil, err } @@ -740,23 +740,23 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI }, nil } -func (s *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { +func (g *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { if len(groupIDs) == 0 { return nil, nil } - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { @@ -771,12 +771,12 @@ func (s *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]* }), nil } -func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { +func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { if !datautil.Contain(req.HandleResult, constant.GroupResponseAgree, constant.GroupResponseRefuse) { return nil, errs.ErrArgs.WrapMsg("HandleResult unknown") } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - groupMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + groupMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -784,11 +784,11 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") } } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } - groupRequest, err := s.db.TakeGroupRequest(ctx, req.GroupID, req.FromUserID) + groupRequest, err := g.db.TakeGroupRequest(ctx, req.GroupID, req.FromUserID) if err != nil { return nil, err } @@ -796,12 +796,12 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup return nil, servererrs.ErrGroupRequestHandled.WrapMsg("group request already processed") } var inGroup bool - if _, err := s.db.TakeGroupMember(ctx, req.GroupID, req.FromUserID); err == nil { + if _, err := g.db.TakeGroupMember(ctx, req.GroupID, req.FromUserID); err == nil { inGroup = true // Already in group - } else if !s.IsNotFound(err) { + } else if !g.IsNotFound(err) { return nil, err } - if _, err := s.user.GetPublicUserInfo(ctx, req.FromUserID); err != nil { + if _, err := g.user.GetPublicUserInfo(ctx, req.FromUserID); err != nil { return nil, err } var member *model.GroupMember @@ -819,37 +819,37 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup OperatorUserID: mcontext.GetOpUserID(ctx), } - if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{member}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{member}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } } log.ZDebug(ctx, "GroupApplicationResponse", "inGroup", inGroup, "HandleResult", req.HandleResult, "member", member) - if err := s.db.HandlerGroupRequest(ctx, req.GroupID, req.FromUserID, req.HandledMsg, req.HandleResult, member); err != nil { + if err := g.db.HandlerGroupRequest(ctx, req.GroupID, req.FromUserID, req.HandledMsg, req.HandleResult, member); err != nil { return nil, err } switch req.HandleResult { case constant.GroupResponseAgree: - s.notification.GroupApplicationAcceptedNotification(ctx, req) + g.notification.GroupApplicationAcceptedNotification(ctx, req) if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { return nil, err } } case constant.GroupResponseRefuse: - s.notification.GroupApplicationRejectedNotification(ctx, req) + g.notification.GroupApplicationRejectedNotification(ctx, req) } return &pbgroup.GroupApplicationResponseResp{}, nil } -func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { - user, err := s.user.GetUserInfo(ctx, req.InviterUserID) +func (g *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { + user, err := g.user.GetUserInfo(ctx, req.InviterUserID) if err != nil { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -865,14 +865,14 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) Ex: req.Ex, } - if err := s.webhookBeforeApplyJoinGroup(ctx, &s.config.WebhooksConfig.BeforeApplyJoinGroup, reqCall); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeApplyJoinGroup(ctx, &g.config.WebhooksConfig.BeforeApplyJoinGroup, reqCall); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - _, err = s.db.TakeGroupMember(ctx, req.GroupID, req.InviterUserID) + _, err = g.db.TakeGroupMember(ctx, req.GroupID, req.InviterUserID) if err == nil { return nil, errs.ErrArgs.Wrap() - } else if !s.IsNotFound(err) && errs.Unwrap(err) != errs.ErrRecordNotFound { + } else if !g.IsNotFound(err) && errs.Unwrap(err) != errs.ErrRecordNotFound { return nil, err } log.ZDebug(ctx, "JoinGroup.groupInfo", "group", group, "eq", group.NeedVerification == constant.Directly) @@ -887,18 +887,18 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMembersJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{groupMember}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{groupMember}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if err := s.db.CreateGroup(ctx, nil, []*model.GroupMember{groupMember}); err != nil { + if err := g.db.CreateGroup(ctx, nil, []*model.GroupMember{groupMember}); err != nil { return nil, err } - if err = s.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID); err != nil { + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID); err != nil { return nil, err } - s.webhookAfterJoinGroup(ctx, &s.config.WebhooksConfig.AfterJoinGroup, req) + g.webhookAfterJoinGroup(ctx, &g.config.WebhooksConfig.AfterJoinGroup, req) return &pbgroup.JoinGroupResp{}, nil } @@ -912,74 +912,74 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) HandledTime: time.Unix(0, 0), Ex: req.Ex, } - if err = s.db.CreateGroupRequest(ctx, []*model.GroupRequest{&groupRequest}); err != nil { + if err = g.db.CreateGroupRequest(ctx, []*model.GroupRequest{&groupRequest}); err != nil { return nil, err } - s.notification.JoinGroupApplicationNotification(ctx, req) + g.notification.JoinGroupApplicationNotification(ctx, req) return &pbgroup.JoinGroupResp{}, nil } -func (s *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) (*pbgroup.QuitGroupResp, error) { +func (g *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) (*pbgroup.QuitGroupResp, error) { if req.UserID == "" { req.UserID = mcontext.GetOpUserID(ctx) } else { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } } - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } if member.RoleLevel == constant.GroupOwner { return nil, errs.ErrNoPermission.WrapMsg("group owner can't quit") } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - err = s.db.DeleteGroupMember(ctx, req.GroupID, []string{req.UserID}) + err = g.db.DeleteGroupMember(ctx, req.GroupID, []string{req.UserID}) if err != nil { return nil, err } - s.notification.MemberQuitNotification(ctx, s.groupMemberDB2PB(member, 0)) - if err := s.deleteMemberAndSetConversationSeq(ctx, req.GroupID, []string{req.UserID}); err != nil { + g.notification.MemberQuitNotification(ctx, g.groupMemberDB2PB(member, 0)) + if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, []string{req.UserID}); err != nil { return nil, err } - s.webhookAfterQuitGroup(ctx, &s.config.WebhooksConfig.AfterQuitGroup, req) + g.webhookAfterQuitGroup(ctx, &g.config.WebhooksConfig.AfterQuitGroup, req) return &pbgroup.QuitGroupResp{}, nil } -func (s *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { +func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conevrsationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := s.msgRpcClient.GetConversationMaxSeq(ctx, conevrsationID) + maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conevrsationID) if err != nil { return err } - return s.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) + return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) } -func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { +func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { var opMember *model.GroupMember - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { var err error - opMember, err = s.db.TakeGroupMember(ctx, req.GroupInfoForSet.GroupID, mcontext.GetOpUserID(ctx)) + opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSet.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } if !(opMember.RoleLevel == constant.GroupOwner || opMember.RoleLevel == constant.GroupAdmin) { return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") } - if err := s.PopulateGroupMember(ctx, opMember); err != nil { + if err := g.PopulateGroupMember(ctx, opMember); err != nil { return nil, err } } - if err := s.webhookBeforeSetGroupInfo(ctx, &s.config.WebhooksConfig.BeforeSetGroupInfo, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeSetGroupInfo(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfo, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) if err != nil { return nil, err } @@ -987,35 +987,35 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return nil, servererrs.ErrDismissedAlready.Wrap() } - count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + count, err := g.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err } - owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + owner, err := g.db.TakeGroupOwner(ctx, group.GroupID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owner); err != nil { + if err := g.PopulateGroupMember(ctx, owner); err != nil { return nil, err } update := UpdateGroupInfoMap(ctx, req.GroupInfoForSet) if len(update) == 0 { return &pbgroup.SetGroupInfoResp{}, nil } - if err := s.db.UpdateGroup(ctx, group.GroupID, update); err != nil { + if err := g.db.UpdateGroup(ctx, group.GroupID, update); err != nil { return nil, err } - group, err = s.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) + group, err = g.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) if err != nil { return nil, err } tips := &sdkws.GroupInfoSetTips{ - Group: s.groupDB2PB(group, owner.UserID, count), + Group: g.groupDB2PB(group, owner.UserID, count), MuteTime: 0, OpUser: &sdkws.GroupMemberFullInfo{}, } if opMember != nil { - tips.OpUser = s.groupMemberDB2PB(opMember, 0) + tips.OpUser = g.groupMemberDB2PB(opMember, 0) } num := len(update) if req.GroupInfoForSet.Notification != "" { @@ -1026,33 +1026,143 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf ConversationType: constant.ReadGroupChatType, GroupID: req.GroupInfoForSet.GroupID, } - resp, err := s.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) if err != nil { log.ZWarn(ctx, "GetGroupMemberIDs", err) return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := s.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) } }() - s.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) } if req.GroupInfoForSet.GroupName != "" { num-- - s.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) } if num > 0 { - s.notification.GroupInfoSetNotification(ctx, tips) + g.notification.GroupInfoSetNotification(ctx, tips) } - s.webhookAfterSetGroupInfo(ctx, &s.config.WebhooksConfig.AfterSetGroupInfo, req) + g.webhookAfterSetGroupInfo(ctx, &g.config.WebhooksConfig.AfterSetGroupInfo, req) return &pbgroup.SetGroupInfoResp{}, nil } -func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) (*pbgroup.TransferGroupOwnerResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupInfoEXReq) (*pbgroup.SetGroupInfoEXResp, error) { + var opMember *model.GroupMember + + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + var err error + + opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSetEX.GroupID, mcontext.GetOpUserID(ctx)) + if err != nil { + return nil, err + } + + if !(opMember.RoleLevel == constant.GroupOwner || opMember.RoleLevel == constant.GroupAdmin) { + return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") + } + + if err := g.PopulateGroupMember(ctx, opMember); err != nil { + return nil, err + } + } + + if err := g.webhookBeforeSetGroupInfoEX(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfoEX, req); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + + group, err := g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } + + count, err := g.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + + owner, err := g.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + + if err := g.PopulateGroupMember(ctx, owner); err != nil { + return nil, err + } + + updatedData := UpdateGroupInfoEXMap(ctx, req.GroupInfoForSetEX) + if len(updatedData) == 0 { + return &pbgroup.SetGroupInfoEXResp{}, nil + } + + if err := g.db.UpdateGroup(ctx, group.GroupID, updatedData); err != nil { + return nil, err + } + + group, err = g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + if err != nil { + return nil, err + } + + tips := &sdkws.GroupInfoSetTips{ + Group: g.groupDB2PB(group, owner.UserID, count), + MuteTime: 0, + OpUser: &sdkws.GroupMemberFullInfo{}, + } + + if opMember != nil { + tips.OpUser = g.groupMemberDB2PB(opMember, 0) + } + + num := len(updatedData) + if req.GroupInfoForSetEX.Notification != nil { + num-- + + func() { + conversation := &pbconversation.ConversationReq{ + ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSetEX.GroupID), + ConversationType: constant.ReadGroupChatType, + GroupID: req.GroupInfoForSetEX.GroupID, + } + + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSetEX.GroupID}) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs", err) + return + } + + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} + + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) + } + }() + + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + } + if req.GroupInfoForSetEX.GroupName != "" { + num-- + + g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) + } + if num > 0 { + g.notification.GroupInfoSetNotification(ctx, tips) + } + + g.webhookAfterSetGroupInfoEX(ctx, &g.config.WebhooksConfig.AfterSetGroupInfoEX, req) + + return &pbgroup.SetGroupInfoEXResp{}, nil +} + +func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) (*pbgroup.TransferGroupOwnerResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -1062,11 +1172,11 @@ func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans if req.OldOwnerUserID == req.NewOwnerUserID { return nil, errs.ErrArgs.WrapMsg("OldOwnerUserID == NewOwnerUserID") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, []string{req.OldOwnerUserID, req.NewOwnerUserID}) + members, err := g.db.FindGroupMembers(ctx, req.GroupID, []string{req.OldOwnerUserID, req.NewOwnerUserID}) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } memberMap := datautil.SliceToMap(members, func(e *model.GroupMember) string { return e.UserID }) @@ -1081,33 +1191,33 @@ func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans if newOwner == nil { return nil, errs.ErrArgs.WrapMsg("NewOwnerUser not in group " + req.NewOwnerUserID) } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if !(mcontext.GetOpUserID(ctx) == oldOwner.UserID && oldOwner.RoleLevel == constant.GroupOwner) { return nil, errs.ErrNoPermission.WrapMsg("no permission transfer group owner") } } - if err := s.db.TransferGroupOwner(ctx, req.GroupID, req.OldOwnerUserID, req.NewOwnerUserID, newOwner.RoleLevel); err != nil { + if err := g.db.TransferGroupOwner(ctx, req.GroupID, req.OldOwnerUserID, req.NewOwnerUserID, newOwner.RoleLevel); err != nil { return nil, err } - s.webhookAfterTransferGroupOwner(ctx, &s.config.WebhooksConfig.AfterTransferGroupOwner, req) + g.webhookAfterTransferGroupOwner(ctx, &g.config.WebhooksConfig.AfterTransferGroupOwner, req) - s.notification.GroupOwnerTransferredNotification(ctx, req) + g.notification.GroupOwnerTransferredNotification(ctx, req) return &pbgroup.TransferGroupOwnerResp{}, nil } -func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) (*pbgroup.GetGroupsResp, error) { +func (g *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) (*pbgroup.GetGroupsResp, error) { var ( group []*model.Group err error ) var resp pbgroup.GetGroupsResp if req.GroupID != "" { - group, err = s.db.FindGroup(ctx, []string{req.GroupID}) + group, err = g.db.FindGroup(ctx, []string{req.GroupID}) resp.Total = uint32(len(group)) } else { var total int64 - total, group, err = s.db.SearchGroup(ctx, req.GroupName, req.Pagination) + total, group, err = g.db.SearchGroup(ctx, req.GroupName, req.Pagination) resp.Total = uint32(total) } @@ -1119,7 +1229,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) return e.GroupID }) - ownerMembers, err := s.db.FindGroupsOwner(ctx, groupIDs) + ownerMembers, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } @@ -1127,7 +1237,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) ownerMemberMap := datautil.SliceToMap(ownerMembers, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } @@ -1145,14 +1255,14 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) return &resp, nil } -func (s *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGroupMembersCMSReq) (*pbgroup.GetGroupMembersCMSResp, error) { - total, members, err := s.db.SearchGroupMember(ctx, req.UserName, req.GroupID, req.Pagination) +func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGroupMembersCMSReq) (*pbgroup.GetGroupMembersCMSResp, error) { + total, members, err := g.db.SearchGroupMember(ctx, req.UserName, req.GroupID, req.Pagination) if err != nil { return nil, err } var resp pbgroup.GetGroupMembersCMSResp resp.Total = uint32(total) - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } resp.Members = datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { @@ -1161,12 +1271,12 @@ func (s *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGr return &resp, nil } -func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { - user, err := s.user.GetPublicUserInfo(ctx, req.UserID) +func (g *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { + user, err := g.user.GetPublicUserInfo(ctx, req.UserID) if err != nil { return nil, err } - total, requests, err := s.db.PageGroupRequestUser(ctx, req.UserID, req.Pagination) + total, requests, err := g.db.PageGroupRequestUser(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -1176,24 +1286,24 @@ func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgrou groupIDs := datautil.Distinct(datautil.Slice(requests, func(e *model.GroupRequest) string { return e.GroupID })) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } groupMap := datautil.SliceToMap(groups, func(e *model.Group) string { return e.GroupID }) - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } @@ -1209,44 +1319,44 @@ func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgrou }, nil } -func (s *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGroupReq) (*pbgroup.DismissGroupResp, error) { - owner, err := s.db.TakeGroupOwner(ctx, req.GroupID) +func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGroupReq) (*pbgroup.DismissGroupResp, error) { + owner, err := g.db.TakeGroupOwner(ctx, req.GroupID) if err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if owner.UserID != mcontext.GetOpUserID(ctx) { return nil, errs.ErrNoPermission.WrapMsg("not group owner") } } - if err := s.PopulateGroupMember(ctx, owner); err != nil { + if err := g.PopulateGroupMember(ctx, owner); err != nil { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } if !req.DeleteMember && group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.WrapMsg("group status is dismissed") } - if err := s.db.DismissGroup(ctx, req.GroupID, req.DeleteMember); err != nil { + if err := g.db.DismissGroup(ctx, req.GroupID, req.DeleteMember); err != nil { return nil, err } if !req.DeleteMember { - num, err := s.db.FindGroupMemberNum(ctx, req.GroupID) + num, err := g.db.FindGroupMemberNum(ctx, req.GroupID) if err != nil { return nil, err } tips := &sdkws.GroupDismissedTips{ - Group: s.groupDB2PB(group, owner.UserID, num), + Group: g.groupDB2PB(group, owner.UserID, num), OpUser: &sdkws.GroupMemberFullInfo{}, } if mcontext.GetOpUserID(ctx) == owner.UserID { - tips.OpUser = s.groupMemberDB2PB(owner, 0) + tips.OpUser = g.groupMemberDB2PB(owner, 0) } - s.notification.GroupDismissedNotification(ctx, tips) + g.notification.GroupDismissedNotification(ctx, tips) } - membersID, err := s.db.FindGroupMemberUserID(ctx, group.GroupID) + membersID, err := g.db.FindGroupMemberUserID(ctx, group.GroupID) if err != nil { return nil, err } @@ -1257,21 +1367,21 @@ func (s *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou GroupType: string(group.GroupType), } - s.webhookAfterDismissGroup(ctx, &s.config.WebhooksConfig.AfterDismissGroup, cbReq) + g.webhookAfterDismissGroup(ctx, &g.config.WebhooksConfig.AfterDismissGroup, cbReq) return &pbgroup.DismissGroupResp{}, nil } -func (s *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGroupMemberReq) (*pbgroup.MuteGroupMemberResp, error) { - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) +func (g *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGroupMemberReq) (*pbgroup.MuteGroupMemberResp, error) { + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - opMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -1289,23 +1399,23 @@ func (s *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGrou } } data := UpdateGroupMemberMutedTimeMap(time.Now().Add(time.Second * time.Duration(req.MutedSeconds))) - if err := s.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { + if err := g.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { return nil, err } - s.notification.GroupMemberMutedNotification(ctx, req.GroupID, req.UserID, req.MutedSeconds) + g.notification.GroupMemberMutedNotification(ctx, req.GroupID, req.UserID, req.MutedSeconds) return &pbgroup.MuteGroupMemberResp{}, nil } -func (s *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.CancelMuteGroupMemberReq) (*pbgroup.CancelMuteGroupMemberResp, error) { - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) +func (g *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.CancelMuteGroupMemberReq) (*pbgroup.CancelMuteGroupMemberResp, error) { + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - opMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -1323,36 +1433,36 @@ func (s *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.Ca } } data := UpdateGroupMemberMutedTimeMap(time.Unix(0, 0)) - if err := s.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { + if err := g.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { return nil, err } - s.notification.GroupMemberCancelMutedNotification(ctx, req.GroupID, req.UserID) + g.notification.GroupMemberCancelMutedNotification(ctx, req.GroupID, req.UserID) return &pbgroup.CancelMuteGroupMemberResp{}, nil } -func (s *groupServer) MuteGroup(ctx context.Context, req *pbgroup.MuteGroupReq) (*pbgroup.MuteGroupResp, error) { - if err := s.CheckGroupAdmin(ctx, req.GroupID); err != nil { +func (g *groupServer) MuteGroup(ctx context.Context, req *pbgroup.MuteGroupReq) (*pbgroup.MuteGroupResp, error) { + if err := g.CheckGroupAdmin(ctx, req.GroupID); err != nil { return nil, err } - if err := s.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupStatusMuted)); err != nil { + if err := g.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupStatusMuted)); err != nil { return nil, err } - s.notification.GroupMutedNotification(ctx, req.GroupID) + g.notification.GroupMutedNotification(ctx, req.GroupID) return &pbgroup.MuteGroupResp{}, nil } -func (s *groupServer) CancelMuteGroup(ctx context.Context, req *pbgroup.CancelMuteGroupReq) (*pbgroup.CancelMuteGroupResp, error) { - if err := s.CheckGroupAdmin(ctx, req.GroupID); err != nil { +func (g *groupServer) CancelMuteGroup(ctx context.Context, req *pbgroup.CancelMuteGroupReq) (*pbgroup.CancelMuteGroupResp, error) { + if err := g.CheckGroupAdmin(ctx, req.GroupID); err != nil { return nil, err } - if err := s.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupOk)); err != nil { + if err := g.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupOk)); err != nil { return nil, err } - s.notification.GroupCancelMutedNotification(ctx, req.GroupID) + g.notification.GroupCancelMutedNotification(ctx, req.GroupID) return &pbgroup.CancelMuteGroupResp{}, nil } -func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGroupMemberInfoReq) (*pbgroup.SetGroupMemberInfoResp, error) { +func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGroupMemberInfoReq) (*pbgroup.SetGroupMemberInfoResp, error) { if len(req.Members) == 0 { return nil, errs.ErrArgs.WrapMsg("members empty") } @@ -1360,7 +1470,7 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if opUserID == "" { return nil, errs.ErrNoPermission.WrapMsg("no op user id") } - isAppManagerUid := authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) for i := range req.Members { req.Members[i].FaceURL = nil } @@ -1390,7 +1500,7 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if _, ok := temp[opUserID]; !ok { userIDs = append(userIDs, opUserID) } - dbMembers, err := s.db.FindGroupMembers(ctx, groupID, userIDs) + dbMembers, err := g.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } @@ -1445,12 +1555,12 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr for i := 0; i < len(req.Members); i++ { - if err := s.webhookBeforeSetGroupMemberInfo(ctx, &s.config.WebhooksConfig.BeforeSetGroupMemberInfo, req.Members[i]); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeSetGroupMemberInfo(ctx, &g.config.WebhooksConfig.BeforeSetGroupMemberInfo, req.Members[i]); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } } - if err := s.db.UpdateGroupMembers(ctx, datautil.Slice(req.Members, func(e *pbgroup.SetGroupMemberInfo) *common.BatchUpdateGroupMember { + if err := g.db.UpdateGroupMembers(ctx, datautil.Slice(req.Members, func(e *pbgroup.SetGroupMemberInfo) *common.BatchUpdateGroupMember { return &common.BatchUpdateGroupMember{ GroupID: e.GroupID, UserID: e.UserID, @@ -1463,30 +1573,30 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if member.RoleLevel != nil { switch member.RoleLevel.Value { case constant.GroupAdmin: - s.notification.GroupMemberSetToAdminNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberSetToAdminNotification(ctx, member.GroupID, member.UserID) case constant.GroupOrdinaryUsers: - s.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) } } if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil { - s.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) } } for i := 0; i < len(req.Members); i++ { - s.webhookAfterSetGroupMemberInfo(ctx, &s.config.WebhooksConfig.AfterSetGroupMemberInfo, req.Members[i]) + g.webhookAfterSetGroupMemberInfo(ctx, &g.config.WebhooksConfig.AfterSetGroupMemberInfo, req.Members[i]) } return &pbgroup.SetGroupMemberInfoResp{}, nil } -func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.GetGroupAbstractInfoReq) (*pbgroup.GetGroupAbstractInfoResp, error) { +func (g *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.GetGroupAbstractInfoReq) (*pbgroup.GetGroupAbstractInfoResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupIDs empty") } if datautil.Duplicate(req.GroupIDs) { return nil, errs.ErrArgs.WrapMsg("groupIDs duplicate") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := g.db.FindGroup(ctx, req.GroupIDs) if err != nil { return nil, err } @@ -1495,7 +1605,7 @@ func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.Get })); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg("not found group " + strings.Join(ids, ",")) } - groupUserMap, err := s.db.MapGroupMemberUserID(ctx, req.GroupIDs) + groupUserMap, err := g.db.MapGroupMemberUserID(ctx, req.GroupIDs) if err != nil { return nil, err } @@ -1510,15 +1620,15 @@ func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.Get }, nil } -func (s *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.GetUserInGroupMembersReq) (*pbgroup.GetUserInGroupMembersResp, error) { +func (g *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.GetUserInGroupMembersReq) (*pbgroup.GetUserInGroupMembersResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupIDs empty") } - members, err := s.db.FindGroupMemberUser(ctx, req.GroupIDs, req.UserID) + members, err := g.db.FindGroupMemberUser(ctx, req.GroupIDs, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } return &pbgroup.GetUserInGroupMembersResp{ @@ -1528,8 +1638,8 @@ func (s *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.Ge }, nil } -func (s *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetGroupMemberUserIDsReq) (*pbgroup.GetGroupMemberUserIDsResp, error) { - userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) +func (g *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetGroupMemberUserIDsReq) (*pbgroup.GetGroupMemberUserIDsResp, error) { + userIDs, err := g.db.FindGroupMemberUserID(ctx, req.GroupID) if err != nil { return nil, err } @@ -1538,15 +1648,15 @@ func (s *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.Ge }, nil } -func (s *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup.GetGroupMemberRoleLevelReq) (*pbgroup.GetGroupMemberRoleLevelResp, error) { +func (g *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup.GetGroupMemberRoleLevelReq) (*pbgroup.GetGroupMemberRoleLevelResp, error) { if len(req.RoleLevels) == 0 { return nil, errs.ErrArgs.WrapMsg("RoleLevels empty") } - members, err := s.db.FindGroupMemberRoleLevels(ctx, req.GroupID, req.RoleLevels) + members, err := g.db.FindGroupMemberRoleLevels(ctx, req.GroupID, req.RoleLevels) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } return &pbgroup.GetGroupMemberRoleLevelResp{ @@ -1556,8 +1666,8 @@ func (s *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup. }, nil } -func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req *pbgroup.GetGroupUsersReqApplicationListReq) (*pbgroup.GetGroupUsersReqApplicationListResp, error) { - requests, err := s.db.FindGroupRequests(ctx, req.GroupID, req.UserIDs) +func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req *pbgroup.GetGroupUsersReqApplicationListReq) (*pbgroup.GetGroupUsersReqApplicationListResp, error) { + requests, err := g.db.FindGroupRequests(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err } @@ -1567,7 +1677,7 @@ func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * groupIDs := datautil.Distinct(datautil.Slice(requests, func(e *model.GroupRequest) string { return e.GroupID })) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } @@ -1577,17 +1687,17 @@ func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * if ids := datautil.Single(groupIDs, datautil.Keys(groupMap)); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index a6952bd6db..8b22c8f9b7 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,6 +17,11 @@ package user import ( "context" "errors" + "math/rand" + "strings" + "sync" + "time" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -29,10 +34,6 @@ import ( "github.com/openimsdk/protocol/group" friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" - "math/rand" - "strings" - "sync" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -147,41 +148,35 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI return nil, err } s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - //if err != nil { - // return nil, err - //} - //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { - // return nil, err - // } - //} - //for _, friendID := range friends { - // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - //} + s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } + func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err } + if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } + data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) //if err != nil { @@ -199,6 +194,7 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } + return resp, nil } func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index ab393dd36e..89062ee0a0 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -18,7 +18,9 @@ const ( CallbackBeforeInviteJoinGroupCommand = "callbackBeforeInviteJoinGroupCommand" CallbackAfterJoinGroupCommand = "callbackAfterJoinGroupCommand" CallbackAfterSetGroupInfoCommand = "callbackAfterSetGroupInfoCommand" + CallbackAfterSetGroupInfoEXCommand = "callbackAfterSetGroupInfoCommandEX" CallbackBeforeSetGroupInfoCommand = "callbackBeforeSetGroupInfoCommand" + CallbackBeforeSetGroupInfoEXCommand = "callbackBeforeSetGroupInfoEXCommand" CallbackAfterRevokeMsgCommand = "callbackBeforeAfterMsgCommand" CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" diff --git a/pkg/callbackstruct/group.go b/pkg/callbackstruct/group.go index 23a73ebd23..7fefa5b926 100644 --- a/pkg/callbackstruct/group.go +++ b/pkg/callbackstruct/group.go @@ -17,6 +17,7 @@ package callbackstruct import ( "github.com/openimsdk/open-im-server/v3/pkg/apistruct" common "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/wrapperspb" ) type CallbackCommand string @@ -242,3 +243,48 @@ type CallbackAfterSetGroupInfoReq struct { type CallbackAfterSetGroupInfoResp struct { CommonCallbackResp } + +type CallbackBeforeSetGroupInfoEXReq struct { + CallbackCommand `json:"callbackCommand"` + OperationID string `json:"operationID"` + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackBeforeSetGroupInfoEXResp struct { + CommonCallbackResp + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackAfterSetGroupInfoEXReq struct { + CallbackCommand `json:"callbackCommand"` + OperationID string `json:"operationID"` + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackAfterSetGroupInfoEXResp struct { + CommonCallbackResp +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index fc6d3001ff..5261e034ce 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -422,6 +422,8 @@ type Webhooks struct { BeforeInviteUserToGroup BeforeConfig `mapstructure:"beforeInviteUserToGroup"` AfterSetGroupInfo AfterConfig `mapstructure:"afterSetGroupInfo"` BeforeSetGroupInfo BeforeConfig `mapstructure:"beforeSetGroupInfo"` + AfterSetGroupInfoEX AfterConfig `mapstructure:"afterSetGroupInfoEX"` + BeforeSetGroupInfoEX BeforeConfig `mapstructure:"beforeSetGroupInfoEX"` AfterRevokeMsg AfterConfig `mapstructure:"afterRevokeMsg"` BeforeAddBlack BeforeConfig `mapstructure:"beforeAddBlack"` AfterAddFriend AfterConfig `mapstructure:"afterAddFriend"` From 938409b0e4139c36172614140f237780e6363f72 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 26 Aug 2024 15:57:57 +0800 Subject: [PATCH 73/91] fix: set min seq (#2556) * fix:log * fix: set min seq --- go.mod | 2 +- go.sum | 5 +++-- internal/rpc/group/notification.go | 12 ++++++++---- internal/rpc/msg/seq.go | 9 +++++++++ pkg/rpcclient/msg.go | 9 +++++++++ 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2d3534b561..ae30db0565 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.5 + github.com/openimsdk/protocol v0.0.72-alpha.9 github.com/openimsdk/tools v0.0.49-alpha.55 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 30fdacb4b9..242b448236 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,9 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.5 h1:1Xjyx6ivTb782Sm7wMJXCLZP80iXADVo1CySis1rOG0= -github.com/openimsdk/protocol v0.0.72-alpha.5/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.8 h1:MhxSsdxXx2ZaeSLQk4uFftsB5L2rPh1Qup+dURQNzXQ= +github.com/openimsdk/protocol v0.0.72-alpha.8/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 72ca9b8130..44eb6f9e32 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -23,6 +23,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -522,8 +523,11 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err != nil { return err } - err = g.conversationRpcClient.SetConversationMinSeq(ctx, entrantUserID, conversationID, maxSeq) - if err != nil { + if _, err = g.msgRpcClient.SetUserConversationsMinSeq(ctx, &msg.SetUserConversationsMinSeqReq{ + UserIDs: entrantUserID, + ConversationID: conversationID, + Seq: maxSeq, + }); err != nil { return err } } @@ -541,9 +545,9 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err != nil { return err } - tips := &sdkws.MemberEnterTips{Group: group, EntrantUsers: users} + tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) + g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) return nil } diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 1ebec4a719..4d9eb6db9c 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -53,3 +53,12 @@ func (m *msgServer) GetMsgByConversationIDs(ctx context.Context, req *pbmsg.GetM } return &pbmsg.GetMsgByConversationIDsResp{MsgDatas: Msgs}, nil } + +func (m *msgServer) SetUserConversationsMinSeq(ctx context.Context, req *pbmsg.SetUserConversationsMinSeqReq) (*pbmsg.SetUserConversationsMinSeqResp, error) { + for _, userID := range req.UserIDs { + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, map[string]int64{req.ConversationID: req.Seq}); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationsMinSeqResp{}, nil +} diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index da556224fb..72d5aab95b 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -159,6 +159,15 @@ func (m *MessageRpcClient) SendMsg(ctx context.Context, req *msg.SendMsgReq) (*m return resp, nil } +// SetUserConversationsMinSeq set min seq +func (m *MessageRpcClient) SetUserConversationsMinSeq(ctx context.Context, req *msg.SetUserConversationsMinSeqReq) (*msg.SetUserConversationsMinSeqResp, error) { + resp, err := m.Client.SetUserConversationsMinSeq(ctx, req) + if err != nil { + return nil, err + } + return resp, nil +} + // GetMaxSeq retrieves the maximum sequence number from the gRPC client. // Errors during the gRPC call are wrapped to provide additional context. func (m *MessageRpcClient) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { From 86a325f309f0598abf4e5d0a60ee5d893e62c0bd Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:15:05 +0800 Subject: [PATCH 74/91] Fix push (#2559) * fix:log * fix: add log * fix: del return * fix: push config * feat: add push err log and extend push wait time * feat: group config * feat: add log in write binary msg * feat: Modify Prometheus data scraping ports and reserve port space. * feat: change group rpc num * feat: change log * fix: remove quotation mark --- config/openim-api.yml | 2 +- config/openim-msggateway.yml | 2 +- config/openim-msgtransfer.yml | 2 +- config/openim-push.yml | 4 ++-- config/openim-rpc-auth.yml | 2 +- config/openim-rpc-conversation.yml | 2 +- config/openim-rpc-friend.yml | 2 +- config/openim-rpc-group.yml | 4 ++-- config/openim-rpc-msg.yml | 2 +- config/openim-rpc-third.yml | 2 +- config/openim-rpc-user.yml | 2 +- config/prometheus.yml | 26 +++++++++++++------------- internal/msggateway/client.go | 2 ++ internal/push/push_handler.go | 4 ++-- pkg/rpcclient/msg.go | 4 +++- start-config.yml | 2 +- 16 files changed, 34 insertions(+), 30 deletions(-) diff --git a/config/openim-api.yml b/config/openim-api.yml index 78a688fcd6..9f53038d88 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -8,6 +8,6 @@ prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must match the number of api.ports - ports: [ 20113 ] + ports: [ 20502 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index d8068c4435..63332740ff 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -8,7 +8,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20112 ] + ports: [ 20640 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 07a7dc1ab1..e71a218edf 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -3,4 +3,4 @@ prometheus: enable: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # Because four instances have been launched, four ports need to be specified - ports: [ 20108, 20109, 20110, 20111 ] + ports: [ 20600, 20601, 20602, 20603 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 1e55cdee87..70aa5997f5 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -4,13 +4,13 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170 ] + ports: [ 10170, 10171, 10172, 10173 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20107 ] + ports: [ 20670, 20671, 20672, 20673 ] maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 979eca06f4..c55c745b6c 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -10,7 +10,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20106 ] + ports: [ 20660 ] tokenPolicy: # Token validity period, in days diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index d3f8225019..00c9c5aab9 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -10,4 +10,4 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20105 ] + ports: [ 20680 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 8d5869ce81..afac3c5db9 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -10,4 +10,4 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20104 ] + ports: [ 20620 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 5fff8b9fe3..d0243b5fbb 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -1,6 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports @@ -10,7 +10,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20103 ] + ports: [ 20650 ] enableHistoryForNewMembers: true \ No newline at end of file diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 38cc46ecfc..15840c7f3b 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -10,7 +10,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20102 ] + ports: [ 20630 ] # Does sending messages require friend verification diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 408251f4df..512ca391fa 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -10,7 +10,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20101 ] + ports: [ 20690 ] object: diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index ec9bb30dcb..1958bbc6ab 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -10,7 +10,7 @@ prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports - ports: [ 20100 ] + ports: [ 20610 ] diff --git a/config/prometheus.yml b/config/prometheus.yml index c7ce4a489c..627cf94114 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -19,65 +19,65 @@ rule_files: # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: - # The job name is added as a label job=job_name" to any timeseries scraped from this config. + # The job name is added as a label "job=job_name" to any timeseries scraped from this config. # Monitored information captured by prometheus # prometheus fetches application services - job_name: node_exporter static_configs: - - targets: [ internal_ip:20114 ] + - targets: [ internal_ip:20500 ] - job_name: openimserver-openim-api static_configs: - - targets: [ internal_ip:20113 ] + - targets: [ internal_ip:20502 ] labels: namespace: default - job_name: openimserver-openim-msggateway static_configs: - - targets: [ internal_ip:20112 ] + - targets: [ internal_ip:20640 ] labels: namespace: default - job_name: openimserver-openim-msgtransfer static_configs: - - targets: [ internal_ip:20111, internal_ip:20110, internal_ip:20109, internal_ip:20108 ] + - targets: [ internal_ip:20600, internal_ip:20601, internal_ip:20602, internal_ip:20603 ] labels: namespace: default - job_name: openimserver-openim-push static_configs: - - targets: [ internal_ip:20107 ] + - targets: [ internal_ip:20670, internal_ip:20671, internal_ip:20672, internal_ip:20673] labels: namespace: default - job_name: openimserver-openim-rpc-auth static_configs: - - targets: [ internal_ip:20106 ] + - targets: [ internal_ip:20600 ] labels: namespace: default - job_name: openimserver-openim-rpc-conversation static_configs: - - targets: [ internal_ip:20105 ] + - targets: [ internal_ip:20680 ] labels: namespace: default - job_name: openimserver-openim-rpc-friend static_configs: - - targets: [ internal_ip:20104 ] + - targets: [ internal_ip:20620 ] labels: namespace: default - job_name: openimserver-openim-rpc-group static_configs: - - targets: [ internal_ip:20103 ] + - targets: [ internal_ip:20650 ] labels: namespace: default - job_name: openimserver-openim-rpc-msg static_configs: - - targets: [ internal_ip:20102 ] + - targets: [ internal_ip:20630 ] labels: namespace: default - job_name: openimserver-openim-rpc-third static_configs: - - targets: [ internal_ip:20101 ] + - targets: [ internal_ip:20690 ] labels: namespace: default - job_name: openimserver-openim-rpc-user static_configs: - - targets: [ internal_ip:20100 ] + - targets: [ internal_ip:20610 ] labels: namespace: default \ No newline at end of file diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index a4902570a6..dcb15b70da 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -271,11 +271,13 @@ func (c *Client) replyMessage(ctx context.Context, binaryReq *Req, err error, re ErrMsg: errResp.ErrMsg, Data: resp, } + t := time.Now() log.ZDebug(ctx, "gateway reply message", "resp", mReply.String()) err = c.writeBinaryMsg(mReply) if err != nil { log.ZWarn(ctx, "wireBinaryMsg replyMessage", err, "resp", mReply.String()) } + log.ZDebug(ctx, "wireBinaryMsg end", "time cost", time.Since(t)) if binaryReq.ReqIdentifier == WsLogoutMsg { return errs.New("user logout", "operationID", binaryReq.OperationID).Wrap() diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 8ecb3dad1a..79d3a92966 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -91,9 +91,9 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { } sec := msgFromMQ.MsgData.SendTime / 1000 nowSec := timeutil.GetCurrentTimestampBySecond() - log.ZDebug(ctx, "push msg", "msg", pbData.String(), "sec", sec, "nowSec", nowSec) + if nowSec-sec > 10 { - return + log.ZWarn(ctx, "long time push msg", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) } var err error switch msgFromMQ.MsgData.SessionType { diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 72d5aab95b..958cb69a62 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -355,7 +355,9 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co } func (s *NotificationSender) NotificationWithSessionType(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) { - s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) }) + if err := s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) }); err != nil { + log.ZWarn(ctx, "Push to queue failed", err, "sendID", sendID, "recvID", recvID, "msg", jsonutil.StructToJsonString(m)) + } } func (s *NotificationSender) Notification(ctx context.Context, sendID, recvID string, contentType int32, m proto.Message, opts ...NotificationOptions) { diff --git a/start-config.yml b/start-config.yml index 21436d7a9a..a6d3e47afa 100644 --- a/start-config.yml +++ b/start-config.yml @@ -3,7 +3,7 @@ serviceBinaries: openim-crontask: 1 openim-rpc-user: 1 openim-msggateway: 1 - openim-push: 1 + openim-push: 4 openim-msgtransfer: 4 openim-rpc-conversation: 1 openim-rpc-auth: 1 From 85614da36ff0eb925d91191ee20f5f689375820e Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:45:44 +0800 Subject: [PATCH 75/91] fix: read seq is written to mongo, online status redis cluster is supported (#2558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo --------- Co-authored-by: withchao --- go.sum | 3 +- .../msgtransfer/online_history_msg_handler.go | 57 +++++++++++++++++++ .../online_msg_to_mongo_handler.go | 1 - internal/push/a_test.go | 29 ++++++++++ pkg/apistruct/msg_test.go | 1 + pkg/common/storage/cache/redis/online.go | 26 ++++++--- pkg/common/storage/cache/redis/online_test.go | 51 +++++++++++++++++ pkg/common/storage/cache/redis/seq_user.go | 24 ++++---- pkg/common/storage/cache/seq_user.go | 1 + pkg/common/storage/controller/msg.go | 5 ++ pkg/common/storage/database/mgo/seq_user.go | 7 +++ 11 files changed, 182 insertions(+), 23 deletions(-) create mode 100644 internal/push/a_test.go create mode 100644 pkg/apistruct/msg_test.go create mode 100644 pkg/common/storage/cache/redis/online_test.go diff --git a/go.sum b/go.sum index 242b448236..815b6badfc 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,7 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.8 h1:MhxSsdxXx2ZaeSLQk4uFftsB5L2rPh1Qup+dURQNzXQ= -github.com/openimsdk/protocol v0.0.72-alpha.8/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.9 h1:Dyx4vs88IU4rJ2YcP/TdYp4ww8JjsMkV89hB/Eazx+A= github.com/openimsdk/protocol v0.0.72-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 77161202cb..8c1978d48b 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -16,6 +16,7 @@ package msgtransfer import ( "context" + "encoding/json" "errors" "github.com/IBM/sarama" "github.com/go-redis/redis" @@ -89,6 +90,7 @@ func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database cont och.conversationRpcClient = conversationRpcClient och.groupRpcClient = groupRpcClient och.historyConsumerGroup = historyConsumerGroup + return &och, err } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { @@ -97,6 +99,7 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID ctx = withAggregationCtx(ctx, ctxMessages) log.ZInfo(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMessages), "key", val.Key()) + och.doSetReadSeq(ctx, ctxMessages) storageMsgList, notStorageMsgList, storageNotificationList, notStorageNotificationList := och.categorizeMessageLists(ctxMessages) @@ -110,6 +113,60 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID och.handleNotification(ctx, val.Key(), conversationIDNotification, storageNotificationList, notStorageNotificationList) } +func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, msgs []*ContextMsg) { + type seqKey struct { + conversationID string + userID string + } + var readSeq map[seqKey]int64 + for _, msg := range msgs { + if msg.message.ContentType != constant.HasReadReceipt { + continue + } + var elem sdkws.NotificationElem + if err := json.Unmarshal(msg.message.Content, &elem); err != nil { + log.ZError(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) + continue + } + var tips sdkws.MarkAsReadTips + if err := json.Unmarshal([]byte(elem.Detail), &tips); err != nil { + log.ZError(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) + continue + } + if len(tips.Seqs) > 0 { + for _, seq := range tips.Seqs { + if tips.HasReadSeq < seq { + tips.HasReadSeq = seq + } + } + clear(tips.Seqs) + tips.Seqs = nil + } + if tips.HasReadSeq < 0 { + continue + } + if readSeq == nil { + readSeq = make(map[seqKey]int64) + } + key := seqKey{ + conversationID: tips.ConversationID, + userID: tips.MarkAsReadUserID, + } + if readSeq[key] > tips.HasReadSeq { + continue + } + readSeq[key] = tips.HasReadSeq + } + if readSeq == nil { + return + } + for key, seq := range readSeq { + if err := och.msgDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { + log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) + } + } +} + func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg { var ctxMessages []*ContextMsg for i := 0; i < len(consumerMessages); i++ { diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index a72fb4792b..cea47fcd50 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -16,7 +16,6 @@ package msgtransfer import ( "context" - "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" diff --git a/internal/push/a_test.go b/internal/push/a_test.go new file mode 100644 index 0000000000..8b2d864071 --- /dev/null +++ b/internal/push/a_test.go @@ -0,0 +1,29 @@ +package push + +import ( + "github.com/openimsdk/protocol/sdkws" + "testing" +) + +func TestName(t *testing.T) { + var c ConsumerHandler + c.readCh = make(chan *sdkws.MarkAsReadTips) + + go c.loopRead() + + go func() { + for i := 0; ; i++ { + seq := int64(i + 1) + if seq%3 == 0 { + seq = 1 + } + c.readCh <- &sdkws.MarkAsReadTips{ + ConversationID: "c100", + MarkAsReadUserID: "u100", + HasReadSeq: seq, + } + } + }() + + select {} +} diff --git a/pkg/apistruct/msg_test.go b/pkg/apistruct/msg_test.go new file mode 100644 index 0000000000..28f878a9fd --- /dev/null +++ b/pkg/apistruct/msg_test.go @@ -0,0 +1 @@ +package apistruct diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index a012e1cd2d..ee1db7e23b 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -8,6 +8,7 @@ import ( "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "strconv" + "strings" "time" ) @@ -66,11 +67,10 @@ func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, o local change = (num1 ~= num2) or (num2 ~= num3) if change then local members = redis.call("ZRANGE", key, 0, -1) - table.insert(members, KEYS[2]) - redis.call("PUBLISH", KEYS[3], table.concat(members, ":")) - return 1 + table.insert(members, "1") + return members else - return 0 + return {"0"} end ` now := time.Now() @@ -82,12 +82,24 @@ func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, o for _, platformID := range online { argv = append(argv, platformID) } - keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} - status, err := s.rdb.Eval(ctx, script, keys, argv).Result() + keys := []string{s.getUserOnlineKey(userID)} + platformIDs, err := s.rdb.Eval(ctx, script, keys, argv).StringSlice() if err != nil { log.ZError(ctx, "redis SetUserOnline", err, "userID", userID, "online", online, "offline", offline) return err } - log.ZDebug(ctx, "redis SetUserOnline", "userID", userID, "online", online, "offline", offline, "status", status) + if len(platformIDs) == 0 { + return errs.ErrInternalServer.WrapMsg("SetUserOnline redis lua invalid return value") + } + if platformIDs[len(platformIDs)-1] != "0" { + log.ZDebug(ctx, "redis SetUserOnline push", "userID", userID, "online", online, "offline", offline, "platformIDs", platformIDs[:len(platformIDs)-1]) + platformIDs[len(platformIDs)-1] = userID + msg := strings.Join(platformIDs, ":") + if err := s.rdb.Publish(ctx, s.channelName, msg).Err(); err != nil { + return errs.Wrap(err) + } + } else { + log.ZDebug(ctx, "redis SetUserOnline not push", "userID", userID, "online", online, "offline", offline) + } return nil } diff --git a/pkg/common/storage/cache/redis/online_test.go b/pkg/common/storage/cache/redis/online_test.go new file mode 100644 index 0000000000..0306f6f5d7 --- /dev/null +++ b/pkg/common/storage/cache/redis/online_test.go @@ -0,0 +1,51 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/redisutil" + "testing" + "time" +) + +/* +address: [ 172.16.8.48:7001, 172.16.8.48:7002, 172.16.8.48:7003, 172.16.8.48:7004, 172.16.8.48:7005, 172.16.8.48:7006 ] +username: +password: passwd123 +clusterMode: true +db: 0 +maxRetry: 10 +*/ +func TestName111111(t *testing.T) { + conf := config.Redis{ + Address: []string{ + "172.16.8.124:7001", + "172.16.8.124:7002", + "172.16.8.124:7003", + "172.16.8.124:7004", + "172.16.8.124:7005", + "172.16.8.124:7006", + }, + ClusterMode: true, + Password: "passwd123", + //Address: []string{"localhost:16379"}, + //Password: "openIM123", + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*1000) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, conf.Build()) + if err != nil { + panic(err) + } + online := NewUserOnline(rdb) + + userID := "a123456" + t.Log(online.GetOnline(ctx, userID)) + t.Log(online.SetUserOnline(ctx, userID, []int32{1, 2, 3, 4}, nil)) + t.Log(online.GetOnline(ctx, userID)) + +} + +func TestName111(t *testing.T) { + +} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index edbc66b21b..0cedfeee12 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -74,17 +74,22 @@ func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID s } func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { - if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { - return err - } + dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) + if err != nil { + return err } - if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { - return errs.Wrap(err) + if dbSeq < seq { + if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + return errs.Wrap(err) + } } return nil } +func (s *seqUserCacheRedis) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq) +} + func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { keys := make([]string, 0, len(seqs)) for conversationID, seq := range seqs { @@ -128,13 +133,6 @@ func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil { return err } - for conversationID, seq := range seqs { - if seq%s.readSeqWriteRatio == 0 { - if err := s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - } - } return nil } diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go index 61dbc0ab45..cef414e16e 100644 --- a/pkg/common/storage/cache/seq_user.go +++ b/pkg/common/storage/cache/seq_user.go @@ -9,6 +9,7 @@ type SeqUser interface { SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8eb417f939..7f884165de 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -77,6 +77,7 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error @@ -808,6 +809,10 @@ func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, c return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } +func (db *commonMsgDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { + return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +} + func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 9faad416ae..244de30000 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -115,5 +115,12 @@ func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conve } func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) + if err != nil { + return err + } + if dbSeq > seq { + return nil + } return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } From bcd5324b48ca93265cd0bf811bfaf9afc50d50b1 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:21:14 +0800 Subject: [PATCH 76/91] fix: invitation to join group notification opuser is null (#2562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification --------- Co-authored-by: withchao --- internal/rpc/group/notification.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 44eb6f9e32..e87e7c495b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -535,7 +535,28 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, entrantUserID); err != nil { return err } - + opUserID := mcontext.GetOpUserID(ctx) + var opUser *sdkws.GroupMemberFullInfo + if authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + opUser = &sdkws.GroupMemberFullInfo{ + GroupID: groupID, + UserID: opUserID, + AppMangerLevel: constant.AppAdmin, + } + } else { + users, err := g.getGroupMembers(ctx, groupID, []string{opUserID}) + if err != nil { + return err + } + if len(users) == 0 { + opUser = &sdkws.GroupMemberFullInfo{ + GroupID: groupID, + UserID: opUserID, + } + } else { + opUser = users[0] + } + } var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, groupID) if err != nil { @@ -545,7 +566,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err != nil { return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} + tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users, OpUser: opUser} g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) return nil From bb4cbcbc78bf2939ed29896fab7b92d6b31f14f1 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:58:36 +0800 Subject: [PATCH 77/91] Fix set convsation (#2564) * fix: set conversation unequal check * fix: check update list --- internal/rpc/conversation/conversaion.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index cb6546f9b0..6f77164e38 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -293,49 +293,48 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver unequal := len(m) if req.Conversation.RecvMsgOpt != nil { - if req.Conversation.RecvMsgOpt.Value != conversationMap[userID].RecvMsgOpt { + if req.Conversation.RecvMsgOpt.Value == conversationMap[userID].RecvMsgOpt { unequal-- } } if req.Conversation.AttachedInfo != nil { - if req.Conversation.AttachedInfo.Value != conversationMap[userID].AttachedInfo { + if req.Conversation.AttachedInfo.Value == conversationMap[userID].AttachedInfo { unequal-- } } if req.Conversation.Ex != nil { - if req.Conversation.Ex.Value != conversationMap[userID].Ex { + if req.Conversation.Ex.Value == conversationMap[userID].Ex { unequal-- } } if req.Conversation.IsPinned != nil { - m["is_pinned"] = req.Conversation.IsPinned.Value - if req.Conversation.IsPinned.Value != conversationMap[userID].IsPinned { + if req.Conversation.IsPinned.Value == conversationMap[userID].IsPinned { unequal-- } } if req.Conversation.GroupAtType != nil { - if req.Conversation.GroupAtType.Value != conversationMap[userID].GroupAtType { + if req.Conversation.GroupAtType.Value == conversationMap[userID].GroupAtType { unequal-- } } if req.Conversation.MsgDestructTime != nil { - if req.Conversation.MsgDestructTime.Value != conversationMap[userID].MsgDestructTime { + if req.Conversation.MsgDestructTime.Value == conversationMap[userID].MsgDestructTime { unequal-- } } if req.Conversation.IsMsgDestruct != nil { - if req.Conversation.IsMsgDestruct.Value != conversationMap[userID].IsMsgDestruct { + if req.Conversation.IsMsgDestruct.Value == conversationMap[userID].IsMsgDestruct { unequal-- } } if req.Conversation.BurnDuration != nil { - if req.Conversation.BurnDuration.Value != conversationMap[userID].BurnDuration { + if req.Conversation.BurnDuration.Value == conversationMap[userID].BurnDuration { unequal-- } } @@ -363,7 +362,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } } else { - if len(m) != 0 { + if len(m) != 0 && len(needUpdateUsersList) != 0 { if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { return nil, err } From c37e0227ffdccf1cfdc983af9ef3eae1f586a227 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 29 Aug 2024 12:17:19 +0800 Subject: [PATCH 78/91] fix: delay deleteObject func. (#2566) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. --- internal/tools/cron_task.go | 41 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index afbaf34b43..337272d69d 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -25,7 +25,6 @@ import ( pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" @@ -59,10 +58,10 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) - if err != nil { - return err - } + // thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + // if err != nil { + // return err + // } conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) if err != nil { @@ -71,7 +70,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { msgClient := msg.NewMsgClient(msgConn) conversationClient := pbconversation.NewConversationClient(conversationConn) - thirdClient := third.NewThirdClient(thirdConn) + // thirdClient := third.NewThirdClient(thirdConn) crontab := cron.New() @@ -115,21 +114,21 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return errs.Wrap(err) } - // scheduled delete outdated file Objects and their datas in specific time. - deleteObjectFunc := func() { - now := time.Now() - deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - return - } - log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - return errs.Wrap(err) - } + // // scheduled delete outdated file Objects and their datas in specific time. + // deleteObjectFunc := func() { + // now := time.Now() + // deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + // ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + // log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + // if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { + // log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + // return + // } + // log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + // } + // if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { + // return errs.Wrap(err) + // } log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() From 275491a1b5c10abb008afb79bf6c0665ac2d6628 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:18:23 +0800 Subject: [PATCH 79/91] fix: memory queue optimization (#2568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size --------- Co-authored-by: withchao --- internal/rpc/relation/friend.go | 2 +- pkg/common/storage/controller/friend.go | 2 +- pkg/rpcclient/msg.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 3d29ad3379..6b52181b6c 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -121,7 +121,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), - queue: memamq.NewMemoryQueue(128, 1024*8), + queue: memamq.NewMemoryQueue(16, 1024*1024), }) return nil } diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 94cb7d661d..88a5fc863d 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -160,7 +160,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, if err != nil { return err } - opUserID := mcontext.GetOperationID(ctx) + opUserID := mcontext.GetOpUserID(ctx) friends := make([]*model.Friend, 0, len(friendUserIDs)*2) myFriendsSet := datautil.SliceSetAny(myFriends, func(friend *model.Friend) string { return friend.FriendUserID diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 958cb69a62..715014800b 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -23,7 +23,6 @@ import ( "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/idutil" @@ -270,8 +269,8 @@ func WithUserRpcClient(userRpcClient *UserRpcClient) NotificationSenderOptions { } const ( - notificationWorkerCount = 2 - notificationBufferSize = 200 + notificationWorkerCount = 16 + notificationBufferSize = 1024 * 1024 * 2 ) func NewNotificationSender(conf *config.Notification, opts ...NotificationSenderOptions) *NotificationSender { @@ -298,7 +297,8 @@ func WithRpcGetUserName() NotificationOptions { } func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) { - ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) + //ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) + ctx = context.WithoutCancel(ctx) ctx, cancel := context.WithTimeout(ctx, time.Second*time.Duration(5)) defer cancel() n := sdkws.NotificationElem{Detail: jsonutil.StructToJsonString(m)} From 8c7de0416bea15f0f6758e796ab20e169e4b1eae Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 30 Aug 2024 17:19:41 +0800 Subject: [PATCH 80/91] fix: fill opUser in invite tips (#2578) * fix: fill opUser in invite tips * fix: del code --- internal/rpc/group/notification.go | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index e87e7c495b..4a69b6aed2 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -535,28 +535,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, entrantUserID); err != nil { return err } - opUserID := mcontext.GetOpUserID(ctx) - var opUser *sdkws.GroupMemberFullInfo - if authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { - opUser = &sdkws.GroupMemberFullInfo{ - GroupID: groupID, - UserID: opUserID, - AppMangerLevel: constant.AppAdmin, - } - } else { - users, err := g.getGroupMembers(ctx, groupID, []string{opUserID}) - if err != nil { - return err - } - if len(users) == 0 { - opUser = &sdkws.GroupMemberFullInfo{ - GroupID: groupID, - UserID: opUserID, - } - } else { - opUser = users[0] - } - } + var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, groupID) if err != nil { @@ -566,7 +545,11 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if err != nil { return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users, OpUser: opUser} + + tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} + if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + return nil + } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) return nil From a5292bb3a39989f0b53de3dcaee5bdd4b5277644 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 3 Sep 2024 18:26:28 +0800 Subject: [PATCH 81/91] feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. --- internal/rpc/group/group.go | 43 ++++++++++++++++++--------------- internal/rpc/relation/friend.go | 4 +++ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9589ed2494..e3f83cdbf5 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1028,12 +1028,12 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf } resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) @@ -1125,33 +1125,36 @@ func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupI if req.GroupInfoForSetEX.Notification != nil { num-- - func() { - conversation := &pbconversation.ConversationReq{ - ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSetEX.GroupID), - ConversationType: constant.ReadGroupChatType, - GroupID: req.GroupInfoForSetEX.GroupID, - } + if req.GroupInfoForSetEX.Notification.Value != "" { + func() { + conversation := &pbconversation.ConversationReq{ + ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSetEX.GroupID), + ConversationType: constant.ReadGroupChatType, + GroupID: req.GroupInfoForSetEX.GroupID, + } - resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSetEX.GroupID}) - if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) - return - } + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSetEX.GroupID}) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) + return + } - conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) - } - }() + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) + } + }() - g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + } } + if req.GroupInfoForSetEX.GroupName != "" { num-- - g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) } + if num > 0 { g.notification.GroupInfoSetNotification(ctx, tips) } diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 6b52181b6c..9130589321 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -312,16 +312,20 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err } + resp.Total = int32(total) + return resp, nil } From 38a880210769d0423a81ff28757be151bcbe69c0 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:32:40 +0800 Subject: [PATCH 82/91] feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log --- config/redis.yml | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/msgtransfer/online_history_msg_handler.go | 7 ++++++- internal/push/offlinepush/dummy/push.go | 2 ++ internal/push/push_handler.go | 3 ++- pkg/common/cmd/root.go | 3 ++- pkg/common/config/config.go | 4 +++- pkg/common/prommetrics/grpc_push.go | 4 ++++ pkg/common/prommetrics/rpc.go | 12 ++++++++++-- pkg/common/storage/cache/redis/batch_handler.go | 2 +- pkg/rpccache/conversation.go | 2 +- 12 files changed, 35 insertions(+), 11 deletions(-) diff --git a/config/redis.yml b/config/redis.yml index 83e3054590..2448bcb5c6 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -4,3 +4,4 @@ password: openIM123 clusterMode: false db: 0 maxRetry: 10 +poolSize: 100 diff --git a/go.mod b/go.mod index ae30db0565..9723208e46 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.9 - github.com/openimsdk/tools v0.0.49-alpha.55 + github.com/openimsdk/tools v0.0.50-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 815b6badfc..1a5f779992 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.9 h1:Dyx4vs88IU4rJ2YcP/TdYp4ww8JjsMkV89hB/Eazx+A= github.com/openimsdk/protocol v0.0.72-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= -github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDITv40Apwq/Tc= +github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 8c1978d48b..6de07cfbce 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -237,6 +237,10 @@ func (och *OnlineHistoryRedisConsumerHandler) categorizeMessageLists(totalMsgs [ } func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key, conversationID string, storageList, notStorageList []*ContextMsg) { + for _, storageMsg := range storageList { + log.ZDebug(ctx, "handle storage msg", "msg", storageMsg.message.String()) + } + och.toPushTopic(ctx, key, conversationID, notStorageList) var storageMessageList []*sdkws.MsgData for _, msg := range storageList { @@ -311,8 +315,9 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con } } -func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(_ context.Context, key, conversationID string, msgs []*ContextMsg) { +func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { + log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) och.msgDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) } } diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 028e7edd34..5698b72947 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -17,6 +17,7 @@ package dummy import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/tools/log" ) func NewClient() *Dummy { @@ -27,5 +28,6 @@ type Dummy struct { } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { + log.ZInfo(ctx, "dummy push") return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 79d3a92966..5d03599944 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -93,7 +93,8 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { nowSec := timeutil.GetCurrentTimestampBySecond() if nowSec-sec > 10 { - log.ZWarn(ctx, "long time push msg", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) + prommetrics.MsgLoneTimePushCounter.Inc() + log.ZWarn(ctx, "it’s been a while since the message was sent", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) } var err error switch msgFromMQ.MsgData.SessionType { diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index b43f86557f..5edea43773 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -129,10 +129,11 @@ func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { } func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { - err := log.InitFromConfig( + err := log.InitLoggerFromConfig( cmdOpts.loggerPrefixName, r.processName, + "", "", r.log.RemainLogLevel, r.log.IsStdout, r.log.IsJson, diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 5261e034ce..8bd16178dd 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -336,7 +336,8 @@ type Redis struct { Password string `mapstructure:"password"` ClusterMode bool `mapstructure:"clusterMode"` DB int `mapstructure:"storage"` - MaxRetry int `mapstructure:"MaxRetry"` + MaxRetry int `mapstructure:"maxRetry"` + PoolSize int `mapstructure:"poolSize"` } type BeforeConfig struct { @@ -474,6 +475,7 @@ func (r *Redis) Build() *redisutil.Config { Password: r.Password, DB: r.DB, MaxRetry: r.MaxRetry, + PoolSize: r.PoolSize, } } diff --git a/pkg/common/prommetrics/grpc_push.go b/pkg/common/prommetrics/grpc_push.go index 0b6c3e76f3..5c966310f7 100644 --- a/pkg/common/prommetrics/grpc_push.go +++ b/pkg/common/prommetrics/grpc_push.go @@ -23,4 +23,8 @@ var ( Name: "msg_offline_push_failed_total", Help: "The number of msg failed offline pushed", }) + MsgLoneTimePushCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "msg_long_time_push_total", + Help: "The number of messages with a push time exceeding 10 seconds", + }) ) diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index dc16322dab..7162fa7e80 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -47,9 +47,17 @@ func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Co case share.RpcRegisterName.MessageGateway: return []prometheus.Collector{OnlineUserGauge} case share.RpcRegisterName.Msg: - return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} + return []prometheus.Collector{ + SingleChatMsgProcessSuccessCounter, + SingleChatMsgProcessFailedCounter, + GroupChatMsgProcessSuccessCounter, + GroupChatMsgProcessFailedCounter, + } case share.RpcRegisterName.Push: - return []prometheus.Collector{MsgOfflinePushFailedCounter} + return []prometheus.Collector{ + MsgOfflinePushFailedCounter, + MsgLoneTimePushCounter, + } case share.RpcRegisterName.Auth: return []prometheus.Collector{UserLoginCounter} case share.RpcRegisterName.User: diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index f9923e198e..1fbd664a32 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -118,7 +118,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { - log.ZError(ctx, "getCache query database failed", err, "key", key) + //log.ZError(ctx, "getCache query database failed", err, "key", key) return "", err } bs, err := json.Marshal(t) diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 2a62c7bbd5..925d2a37ca 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -86,7 +86,7 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co if err == nil { log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "userID", userID, "conversationID", conversationID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) + log.ZWarn(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() var cache cacheProto[pbconversation.Conversation] From 2477a6658cc3e85ea0002acf8bc1e6d0db20f843 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:14:21 +0800 Subject: [PATCH 83/91] feat: supports getting messages based on session ID and seq (#2582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/constant.go | 1 + internal/msggateway/message_handler.go | 20 +++++++++ internal/rpc/group/group.go | 2 +- internal/rpc/group/notification.go | 43 ++++++++++++++----- internal/rpc/msg/sync_msg.go | 36 ++++++++++++++-- pkg/rpcclient/msg.go | 4 ++ pkg/util/conversationutil/conversationutil.go | 8 ++++ 10 files changed, 103 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 9723208e46..76166fa4f1 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.9 + github.com/openimsdk/protocol v0.0.72-alpha.12 github.com/openimsdk/tools v0.0.50-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 1a5f779992..bfd625d49c 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.9 h1:Dyx4vs88IU4rJ2YcP/TdYp4ww8JjsMkV89hB/Eazx+A= -github.com/openimsdk/protocol v0.0.72-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.12 h1:GXUtSFXlh1AeOmMjN1CsRfRZMTQYBWZ8mTuRoB7KxLQ= +github.com/openimsdk/protocol v0.0.72-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDITv40Apwq/Tc= github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index dcb15b70da..5a274f9e7a 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -220,6 +220,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.SendSignalMessage(ctx, binaryReq) case WSPullMsgBySeqList: resp, messageErr = c.longConnServer.PullMessageBySeqList(ctx, binaryReq) + case WSPullMsg: + resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index dc5ad77861..154be014e9 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -39,6 +39,7 @@ const ( WSPullMsgBySeqList = 1002 WSSendMsg = 1003 WSSendSignalMsg = 1004 + WSPullMsg = 1005 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 8a11e6ab3c..2f9620ce16 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -94,6 +94,7 @@ type MessageHandler interface { SendMessage(context context.Context, data *Req) ([]byte, error) SendSignalMessage(context context.Context, data *Req) ([]byte, error) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) + GetSeqMessage(context context.Context, data *Req) ([]byte, error) UserLogout(context context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(context context.Context, data *Req) ([]byte, bool, error) } @@ -191,6 +192,25 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ return c, nil } +func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, error) { + req := msg.GetSeqMessageReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "GetSeqMessage") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetSeqMessage") + } + resp, err := g.msgRpcClient.GetSeqMessage(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetSeqMessage") + } + return c, nil +} + func (g GrpcHandler) UserLogout(context context.Context, data *Req) ([]byte, error) { req := push.DelUserPushTokenReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index e3f83cdbf5..c45a7827b7 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -833,7 +833,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, groupRequest.InviterUserID, req.FromUserID); err != nil { return nil, err } } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 4a69b6aed2..64e922fe2b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -16,27 +16,28 @@ package group import ( "context" + "errors" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" - "github.com/openimsdk/protocol/msg" - - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" + "go.mongodb.org/mongo-driver/mongo" ) // GroupApplicationReceiver @@ -227,10 +228,13 @@ func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, ap } */ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { + return g.fillOpUserByUserID(ctx, mcontext.GetOpUserID(ctx), opUser, groupID) +} + +func (g *GroupNotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { if opUser == nil { return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } - userID := mcontext.GetOpUserID(ctx) if groupID != "" { if authverify.IsManagerUserID(userID, g.config.Share.IMAdminUserID) { *opUser = &sdkws.GroupMemberFullInfo{ @@ -243,7 +247,7 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws member, err := g.db.TakeGroupMember(ctx, groupID, userID) if err == nil { *opUser = g.groupMemberDB2PB(member, 0) - } else if !errs.ErrRecordNotFound.Is(err) { + } else if !(errors.Is(err, mongo.ErrNoDocuments) || errs.ErrRecordNotFound.Is(err)) { return err } } @@ -509,7 +513,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { +func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { @@ -546,15 +550,32 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + tips := &sdkws.MemberInvitedTips{ + Group: group, + InvitedUserList: users, + } + opUserID := mcontext.GetOpUserID(ctx) + if err = g.fillOpUserByUserID(ctx, opUserID, &tips.OpUser, tips.Group.GroupID); err != nil { return nil } + switch { + case invitedOpUserID == "": + case invitedOpUserID == opUserID: + tips.InviterUser = tips.OpUser + default: + if err = g.fillOpUserByUserID(ctx, invitedOpUserID, &tips.InviterUser, tips.Group.GroupID); err != nil { + return err + } + } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) return nil } +func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { + return g.GroupApplicationAgreeMemberEnterNotification(ctx, groupID, "", entrantUserID...) +} + func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { var err error defer func() { diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index f5b5ebda53..0b37b25c28 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -16,16 +16,15 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/timeutil" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/timeutil" ) func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessageBySeqsReq) (*sdkws.PullMessageBySeqsResp, error) { @@ -86,6 +85,35 @@ func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessag return resp, nil } +func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + resp := &msg.GetSeqMessageResp{ + Msgs: make(map[string]*sdkws.PullMsgs), + NotificationMsgs: make(map[string]*sdkws.PullMsgs), + } + for _, conv := range req.Conversations { + _, _, msgs, err := m.MsgDatabase.GetMsgBySeqs(ctx, req.UserID, conv.ConversationID, conv.Seqs) + if err != nil { + return nil, err + } + var pullMsgs *sdkws.PullMsgs + if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { + pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.NotificationMsgs[conv.ConversationID] = pullMsgs + } + } else { + pullMsgs, ok = resp.Msgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.NotificationMsgs[conv.ConversationID] = pullMsgs + } + } + pullMsgs.Msgs = append(pullMsgs.Msgs, msgs...) + } + return resp, nil +} + func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { return nil, err diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 715014800b..5a06dac5db 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -221,6 +221,10 @@ func (m *MessageRpcClient) PullMessageBySeqList(ctx context.Context, req *sdkws. return resp, nil } +func (m *MessageRpcClient) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + return m.Client.GetSeqMessage(ctx, req) +} + func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversationID string) (int64, error) { resp, err := m.Client.GetConversationMaxSeq(ctx, &msg.GetConversationMaxSeqReq{ConversationID: conversationID}) if err != nil { diff --git a/pkg/util/conversationutil/conversationutil.go b/pkg/util/conversationutil/conversationutil.go index 5683d8df89..f0a44ab1e1 100644 --- a/pkg/util/conversationutil/conversationutil.go +++ b/pkg/util/conversationutil/conversationutil.go @@ -19,6 +19,14 @@ func GenGroupConversationID(groupID string) string { return "sg_" + groupID } +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "sg_") +} + +func IsNotificationConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "n_") +} + func GenConversationUniqueKeyForSingle(sendID, recvID string) string { l := []string{sendID, recvID} sort.Strings(l) From 20085d0efcb2b3b5801f3f54275937c4bcd69c39 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 5 Sep 2024 09:53:24 +0800 Subject: [PATCH 84/91] feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 76166fa4f1..45a92fe634 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.12 + github.com/openimsdk/protocol v0.0.72-alpha.13 github.com/openimsdk/tools v0.0.50-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index bfd625d49c..fe587bd1bb 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.12 h1:GXUtSFXlh1AeOmMjN1CsRfRZMTQYBWZ8mTuRoB7KxLQ= -github.com/openimsdk/protocol v0.0.72-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.13 h1:ILpvuxWGrVJMVCPRodOQcrSMFKUBzLahBPb8GkITWSc= +github.com/openimsdk/protocol v0.0.72-alpha.13/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDITv40Apwq/Tc= github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 6dd4e56c2ab20e62e1300ce9e145c6058bb62d29 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:31:00 +0800 Subject: [PATCH 85/91] fix: getting messages based on session ID and seq (#2595) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao --- internal/rpc/msg/sync_msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 0b37b25c28..f13dfb31c0 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -106,7 +106,7 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq pullMsgs, ok = resp.Msgs[conv.ConversationID] if !ok { pullMsgs = &sdkws.PullMsgs{} - resp.NotificationMsgs[conv.ConversationID] = pullMsgs + resp.Msgs[conv.ConversationID] = pullMsgs } } pullMsgs.Msgs = append(pullMsgs.Msgs, msgs...) From eea2627a289bb66b5f9cf08ff946a0001a92b199 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:48:54 +0800 Subject: [PATCH 86/91] feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao --- internal/rpc/msg/sync_msg.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index f13dfb31c0..ea4c487b9e 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -132,6 +132,12 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd log.ZWarn(ctx, "GetMaxSeqs error", err, "conversationIDs", conversationIDs, "maxSeqs", maxSeqs) return nil, err } + // avoid pulling messages from sessions with a large number of max seq values of 0 + for conversationID, seq := range maxSeqs { + if seq == 0 { + delete(maxSeqs, conversationID) + } + } resp := new(sdkws.GetMaxSeqResp) resp.MaxSeqs = maxSeqs return resp, nil From c581d43f171dd6df3d86d38f157183b16e35e05c Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 10 Sep 2024 10:26:02 +0800 Subject: [PATCH 87/91] refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` --- internal/msgtransfer/init.go | 21 +- .../msgtransfer/online_history_msg_handler.go | 25 +- .../online_msg_to_mongo_handler.go | 11 +- pkg/common/storage/controller/msg.go | 139 --------- pkg/common/storage/controller/msg_transfer.go | 286 ++++++++++++++++++ 5 files changed, 317 insertions(+), 165 deletions(-) create mode 100644 pkg/common/storage/controller/msg_transfer.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5e0ccd0e52..e67af85248 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -18,19 +18,20 @@ import ( "context" "errors" "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" - "net/http" - "os" - "os/signal" - "syscall" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" @@ -65,6 +66,7 @@ type Config struct { func Start(ctx context.Context, index int, config *Config) error { log.CInfo(ctx, "MSG-TRANSFER server is initializing", "prometheusPorts", config.MsgTransfer.Prometheus.Ports, "index", index) + mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -73,12 +75,13 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return err } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { @@ -94,17 +97,17 @@ func Start(ctx context.Context, index int, config *Config) error { return err } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) + msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgDatabase, &conversationRpcClient, &groupRpcClient) + historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) if err != nil { return err } - historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgDatabase) + historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgTransferDatabase) if err != nil { return err } diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 6de07cfbce..6b924b05b9 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,6 +18,10 @@ import ( "context" "encoding/json" "errors" + "strconv" + "strings" + "time" + "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -33,9 +37,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/stringutil" "google.golang.org/protobuf/proto" - "strconv" - "strings" - "time" ) const ( @@ -56,19 +57,19 @@ type OnlineHistoryRedisConsumerHandler struct { redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase conversationRpcClient *rpcclient.ConversationRpcClient groupRpcClient *rpcclient.GroupRpcClient } -func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase, +func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*OnlineHistoryRedisConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) if err != nil { return nil, err } var och OnlineHistoryRedisConsumerHandler - och.msgDatabase = database + och.msgTransferDatabase = database b := batcher.New[sarama.ConsumerMessage]( batcher.WithSize(size), @@ -161,7 +162,7 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, return } for key, seq := range readSeq { - if err := och.msgDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) } } @@ -248,7 +249,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } if len(storageMessageList) > 0 { msg := storageMessageList[0] - lastSeq, isNewConversation, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, isNewConversation, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return @@ -282,7 +283,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } log.ZDebug(ctx, "success incr to next topic") - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) @@ -299,14 +300,14 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con storageMessageList = append(storageMessageList, msg.message) } if len(storageMessageList) > 0 { - lastSeq, _, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil { log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageMessageList) return } log.ZDebug(ctx, "success to next topic", "conversationID", conversationID) - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) @@ -318,7 +319,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) - och.msgDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) + och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) } } diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index cea47fcd50..ef6f6ac7d9 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -16,6 +16,7 @@ package msgtransfer import ( "context" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -28,10 +29,10 @@ import ( type OnlineHistoryMongoConsumerHandler struct { historyConsumerGroup *kafka.MConsumerGroup - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase } -func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase) (*OnlineHistoryMongoConsumerHandler, error) { +func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryMongoConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToMongoGroupID, []string{kafkaConf.ToMongoTopic}, true) if err != nil { return nil, err @@ -39,7 +40,7 @@ func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database cont mc := &OnlineHistoryMongoConsumerHandler{ historyConsumerGroup: historyConsumerGroup, - msgDatabase: database, + msgTransferDatabase: database, } return mc, nil } @@ -57,7 +58,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont return } log.ZInfo(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) - err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) + err = mc.msgTransferDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) if err != nil { log.ZError( ctx, @@ -76,7 +77,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont for _, msg := range msgFromMQ.MsgData { seqs = append(seqs, msg.Seq) } - err = mc.msgDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) + err = mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) if err != nil { log.ZError( ctx, diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 7f884165de..fdd06d3ff3 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -26,7 +26,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" @@ -47,16 +46,10 @@ const ( // CommonMsgDatabase defines the interface for message database operations. type CommonMsgDatabase interface { - // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. - BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error // RevokeMsg revokes a message in a conversation. RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error // MarkSingleChatMsgsAsRead marks messages as read for a single chat by sequence numbers. MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error - // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. - DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error - // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. - BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) // GetMsgBySeqsRange retrieves messages from MongoDB by a range of sequence numbers. GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // GetMsgBySeqs retrieves messages for large groups from MongoDB by sequence numbers. @@ -77,7 +70,6 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error @@ -91,8 +83,6 @@ type CommonMsgDatabase interface { // to mq MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error - MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) - MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) @@ -114,22 +104,12 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser if err != nil { return nil, err } - producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) - if err != nil { - return nil, err - } - producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) - if err != nil { - return nil, err - } return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, - producerToMongo: producerToMongo, - producerToPush: producerToPush, }, nil } @@ -140,8 +120,6 @@ type commonMsgDatabase struct { seqConversation cache.SeqConversationCache seqUser cache.SeqUser producer *kafka.Producer - producerToMongo *kafka.Producer - producerToPush *kafka.Producer } func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error { @@ -149,23 +127,6 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd return err } -func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { - partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) - if err != nil { - log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) - return 0, 0, err - } - return partition, offset, nil -} - -func (db *commonMsgDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { - if len(messages) > 0 { - _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) - return err - } - return nil -} - func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { if len(fields) == 0 { return nil @@ -267,52 +228,6 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI return nil } -func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { - if len(msgList) == 0 { - return errs.ErrArgs.WrapMsg("msgList is empty") - } - msgs := make([]any, len(msgList)) - for i, msg := range msgList { - if msg == nil { - continue - } - var offlinePushModel *model.OfflinePushModel - if msg.OfflinePushInfo != nil { - offlinePushModel = &model.OfflinePushModel{ - Title: msg.OfflinePushInfo.Title, - Desc: msg.OfflinePushInfo.Desc, - Ex: msg.OfflinePushInfo.Ex, - IOSPushSound: msg.OfflinePushInfo.IOSPushSound, - IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, - } - } - msgs[i] = &model.MsgDataModel{ - SendID: msg.SendID, - RecvID: msg.RecvID, - GroupID: msg.GroupID, - ClientMsgID: msg.ClientMsgID, - ServerMsgID: msg.ServerMsgID, - SenderPlatformID: msg.SenderPlatformID, - SenderNickname: msg.SenderNickname, - SenderFaceURL: msg.SenderFaceURL, - SessionType: msg.SessionType, - MsgFrom: msg.MsgFrom, - ContentType: msg.ContentType, - Content: string(msg.Content), - Seq: msg.Seq, - SendTime: msg.SendTime, - CreateTime: msg.CreateTime, - Status: msg.Status, - Options: msg.Options, - OfflinePush: offlinePushModel, - AtUserIDList: msg.AtUserIDList, - AttachedInfo: msg.AttachedInfo, - Ex: msg.Ex, - } - } - return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) -} - func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error { return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq) } @@ -332,56 +247,6 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return nil } -func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) -} - -func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { - for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - } - return nil -} - -func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - lenList := len(msgs) - if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { - return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() - } - if lenList < 1 { - return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() - } - currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) - if err != nil { - log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err - } - isNew = currentMaxSeq == 0 - lastMaxSeq := currentMaxSeq - userSeqMap := make(map[string]int64) - for _, m := range msgs { - currentMaxSeq++ - m.Seq = currentMaxSeq - userSeqMap[m.SendID] = m.Seq - } - - failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) - if err != nil { - prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) - log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) - } else { - prommetrics.MsgInsertRedisSuccessCounter.Inc() - } - err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) - if err != nil { - log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - return lastMaxSeq, isNew, errs.Wrap(err) -} - func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { // log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs) @@ -809,10 +674,6 @@ func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, c return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } -func (db *commonMsgDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) -} - func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go new file mode 100644 index 0000000000..5e540a2c33 --- /dev/null +++ b/pkg/common/storage/controller/msg_transfer.go @@ -0,0 +1,286 @@ +package controller + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + pbmsg "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "go.mongodb.org/mongo-driver/mongo" +) + +type MsgTransferDatabase interface { + // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. + BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error + // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. + DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error + + // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. + BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) + SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + + // to mq + MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) + MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error +} + +func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (MsgTransferDatabase, error) { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil, err + } + producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) + if err != nil { + return nil, err + } + producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) + if err != nil { + return nil, err + } + return &msgTransferDatabase{ + msgDocDatabase: msgDocModel, + msg: msg, + seqUser: seqUser, + seqConversation: seqConversation, + producerToMongo: producerToMongo, + producerToPush: producerToPush, + }, nil +} + +type msgTransferDatabase struct { + msgDocDatabase database.Msg + msgTable model.MsgDocModel + msg cache.MsgCache + seqConversation cache.SeqConversationCache + seqUser cache.SeqUser + producerToMongo *kafka.Producer + producerToPush *kafka.Producer +} + +func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { + if len(msgList) == 0 { + return errs.ErrArgs.WrapMsg("msgList is empty") + } + msgs := make([]any, len(msgList)) + for i, msg := range msgList { + if msg == nil { + continue + } + var offlinePushModel *model.OfflinePushModel + if msg.OfflinePushInfo != nil { + offlinePushModel = &model.OfflinePushModel{ + Title: msg.OfflinePushInfo.Title, + Desc: msg.OfflinePushInfo.Desc, + Ex: msg.OfflinePushInfo.Ex, + IOSPushSound: msg.OfflinePushInfo.IOSPushSound, + IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, + } + } + msgs[i] = &model.MsgDataModel{ + SendID: msg.SendID, + RecvID: msg.RecvID, + GroupID: msg.GroupID, + ClientMsgID: msg.ClientMsgID, + ServerMsgID: msg.ServerMsgID, + SenderPlatformID: msg.SenderPlatformID, + SenderNickname: msg.SenderNickname, + SenderFaceURL: msg.SenderFaceURL, + SessionType: msg.SessionType, + MsgFrom: msg.MsgFrom, + ContentType: msg.ContentType, + Content: string(msg.Content), + Seq: msg.Seq, + SendTime: msg.SendTime, + CreateTime: msg.CreateTime, + Status: msg.Status, + Options: msg.Options, + OfflinePush: offlinePushModel, + AtUserIDList: msg.AtUserIDList, + AttachedInfo: msg.AttachedInfo, + Ex: msg.Ex, + } + } + return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) +} + +func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { + if len(fields) == 0 { + return nil + } + num := db.msgTable.GetSingleGocMsgNum() + // num = 100 + for i, field := range fields { // Check the type of the field + var ok bool + switch key { + case updateKeyMsg: + var msg *model.MsgDataModel + msg, ok = field.(*model.MsgDataModel) + if msg != nil && msg.Seq != firstSeq+int64(i) { + return errs.ErrInternalServer.WrapMsg("seq is invalid") + } + case updateKeyRevoke: + _, ok = field.(*model.RevokeModel) + default: + return errs.ErrInternalServer.WrapMsg("key is invalid") + } + if !ok { + return errs.ErrInternalServer.WrapMsg("field type is invalid") + } + } + // Returns true if the document exists in the database, false if the document does not exist in the database + updateMsgModel := func(seq int64, i int) (bool, error) { + var ( + res *mongo.UpdateResult + err error + ) + docID := db.msgTable.GetDocID(conversationID, seq) + index := db.msgTable.GetMsgIndex(seq) + field := fields[i] + switch key { + case updateKeyMsg: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "msg", field) + case updateKeyRevoke: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "revoke", field) + } + if err != nil { + return false, err + } + return res.MatchedCount > 0, nil + } + tryUpdate := true + for i := 0; i < len(fields); i++ { + seq := firstSeq + int64(i) // Current sequence number + if tryUpdate { + matched, err := updateMsgModel(seq, i) + if err != nil { + return err + } + if matched { + continue // The current data has been updated, skip the current data + } + } + doc := model.MsgDocModel{ + DocID: db.msgTable.GetDocID(conversationID, seq), + Msg: make([]*model.MsgInfoModel, num), + } + var insert int // Inserted data number + for j := i; j < len(fields); j++ { + seq = firstSeq + int64(j) + if db.msgTable.GetDocID(conversationID, seq) != doc.DocID { + break + } + insert++ + switch key { + case updateKeyMsg: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Msg: fields[j].(*model.MsgDataModel), + } + case updateKeyRevoke: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Revoke: fields[j].(*model.RevokeModel), + } + } + } + for i, msgInfo := range doc.Msg { + if msgInfo == nil { + msgInfo = &model.MsgInfoModel{} + doc.Msg[i] = msgInfo + } + if msgInfo.DelList == nil { + doc.Msg[i].DelList = []string{} + } + } + if err := db.msgDocDatabase.Create(ctx, &doc); err != nil { + if mongo.IsDuplicateKeyError(err) { + i-- // already inserted + tryUpdate = true // next block use update mode + continue + } + return err + } + tryUpdate = false // The current block is inserted successfully, and the next block is inserted preferentially + i += insert - 1 // Skip the inserted data + } + return nil +} + +func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { + return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) +} + +func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { + lenList := len(msgs) + if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { + return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() + } + if lenList < 1 { + return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() + } + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err + } + isNew = currentMaxSeq == 0 + lastMaxSeq := currentMaxSeq + userSeqMap := make(map[string]int64) + for _, m := range msgs { + currentMaxSeq++ + m.Seq = currentMaxSeq + userSeqMap[m.SendID] = m.Seq + } + + failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) + if err != nil { + prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) + log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) + } else { + prommetrics.MsgInsertRedisSuccessCounter.Inc() + } + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) + if err != nil { + log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) + prommetrics.SeqSetFailedCounter.Inc() + } + return lastMaxSeq, isNew, errs.Wrap(err) +} + +func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil +} + +func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { + return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +} + +func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { + partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) + if err != nil { + log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) + return 0, 0, err + } + return partition, offset, nil +} + +func (db *msgTransferDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { + if len(messages) > 0 { + _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) + if err != nil { + log.ZError(ctx, "MsgToMongoMQ", err, "key", key, "conversationID", conversationID, "lastSeq", lastSeq) + return err + } + } + return nil +} From 3381b85895dc8fbe1bc0da81f082bb333b75f0c0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 10 Sep 2024 19:10:15 +0800 Subject: [PATCH 88/91] feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. --- config/kafka.yml | 4 + go.mod | 2 +- go.sum | 4 +- internal/msgtransfer/init.go | 1 + internal/push/offlinepush/getui/push.go | 13 ++- internal/push/offlinepush_handler.go | 122 ++++++++++++++++++++++++ internal/push/push.go | 19 +++- internal/push/push_handler.go | 82 ++++++++-------- pkg/common/cmd/push.go | 1 - pkg/common/config/config.go | 27 +++--- pkg/common/storage/controller/push.go | 30 +++++- scripts/create-topic.sh | 2 +- tools/check-component/main.go | 13 +-- 13 files changed, 246 insertions(+), 74 deletions(-) create mode 100644 internal/push/offlinepush_handler.go diff --git a/config/kafka.yml b/config/kafka.yml index e45474f275..fd06ae2bb4 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -14,12 +14,16 @@ toRedisTopic: toRedis toMongoTopic: toMongo # Kafka topic for push notifications toPushTopic: toPush +# Kafka topic for offline push notifications +toOfflinePushTopic: toOfflinePush # Consumer group ID for Redis topic toRedisGroupID: redis # Consumer group ID for MongoDB topic toMongoGroupID: mongo # Consumer group ID for push notifications topic toPushGroupID: push +# Consumer group ID for offline push notifications topic +toOfflinePushGroupID: offlinePush # TLS (Transport Layer Security) configuration tls: # Enable or disable TLS diff --git a/go.mod b/go.mod index 45a92fe634..d27ad28ea5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.13 + github.com/openimsdk/protocol v0.0.72-alpha.17 github.com/openimsdk/tools v0.0.50-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index fe587bd1bb..85ce53a62f 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.13 h1:ILpvuxWGrVJMVCPRodOQcrSMFKUBzLahBPb8GkITWSc= -github.com/openimsdk/protocol v0.0.72-alpha.13/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.17 h1:kB7eyjJHdkc8lpSlLIHskHzbodxkIG4eaK908iQLVdI= +github.com/openimsdk/protocol v0.0.72-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDITv40Apwq/Tc= github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index e67af85248..a1c8b26b6c 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -111,6 +111,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } + msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index 27b19e8fe0..674d081160 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -18,11 +18,12 @@ import ( "context" "crypto/sha256" "encoding/hex" - "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "strconv" "sync" "time" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/errs" @@ -91,6 +92,16 @@ func (g *Client) Push(ctx context.Context, userIDs []string, title, content stri for i, v := range s.GetSplitResult() { go func(index int, userIDs []string) { defer wg.Done() + for i := 0; i < len(userIDs); i += maxNum { + end := i + maxNum + if end > len(userIDs) { + end = len(userIDs) + } + if err = g.batchPush(ctx, token, userIDs[i:end], pushReq); err != nil { + log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) + } + + } if err = g.batchPush(ctx, token, userIDs, pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) } diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go new file mode 100644 index 0000000000..e97e0e4db8 --- /dev/null +++ b/internal/push/offlinepush_handler.go @@ -0,0 +1,122 @@ +package push + +import ( + "context" + + "github.com/IBM/sarama" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/protocol/constant" + pbpush "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/utils/jsonutil" + "google.golang.org/protobuf/proto" +) + +type OfflinePushConsumerHandler struct { + OfflinePushConsumerGroup *kafka.MConsumerGroup + offlinePusher offlinepush.OfflinePusher +} + +func NewOfflinePushConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher) (*OfflinePushConsumerHandler, error) { + var offlinePushConsumerHandler OfflinePushConsumerHandler + var err error + offlinePushConsumerHandler.offlinePusher = offlinePusher + offlinePushConsumerHandler.OfflinePushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToOfflineGroupID, + []string{config.KafkaConfig.ToOfflinePushTopic}, true) + if err != nil { + return nil, err + } + return &offlinePushConsumerHandler, nil +} + +func (*OfflinePushConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } +func (*OfflinePushConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } +func (o *OfflinePushConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + ctx := o.OfflinePushConsumerGroup.GetContextFromMsg(msg) + o.handleMsg2OfflinePush(ctx, msg.Value) + sess.MarkMessage(msg, "") + } + return nil +} + +func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, msg []byte) { + offlinePushMsg := pbpush.PushMsgReq{} + if err := proto.Unmarshal(msg, &offlinePushMsg); err != nil { + log.ZError(ctx, "offline push Unmarshal msg err", err, "msg", string(msg)) + return + } + if offlinePushMsg.MsgData == nil || offlinePushMsg.UserIDs == nil { + log.ZError(ctx, "offline push msg is empty", errs.New("offlinePushMsg is empty"), "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + return + } + log.ZInfo(ctx, "receive to OfflinePush MQ", "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + + err := o.offlinePushMsg(ctx, offlinePushMsg.MsgData, offlinePushMsg.UserIDs) + if err != nil { + log.ZWarn(ctx, "offline push failed", err, "msg", offlinePushMsg.String()) + } +} + +func (c *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, content string, opts *options.Opts, err error) { + type AtTextElem struct { + Text string `json:"text,omitempty"` + AtUserList []string `json:"atUserList,omitempty"` + IsAtSelf bool `json:"isAtSelf"` + } + + opts = &options.Opts{Signal: &options.Signal{}} + if msg.OfflinePushInfo != nil { + opts.IOSBadgeCount = msg.OfflinePushInfo.IOSBadgeCount + opts.IOSPushSound = msg.OfflinePushInfo.IOSPushSound + opts.Ex = msg.OfflinePushInfo.Ex + } + + if msg.OfflinePushInfo != nil { + title = msg.OfflinePushInfo.Title + content = msg.OfflinePushInfo.Desc + } + if title == "" { + switch msg.ContentType { + case constant.Text: + fallthrough + case constant.Picture: + fallthrough + case constant.Voice: + fallthrough + case constant.Video: + fallthrough + case constant.File: + title = constant.ContentType2PushContent[int64(msg.ContentType)] + case constant.AtText: + ac := AtTextElem{} + _ = jsonutil.JsonStringToStruct(string(msg.Content), &ac) + case constant.SignalingNotification: + title = constant.ContentType2PushContent[constant.SignalMsg] + default: + title = constant.ContentType2PushContent[constant.Common] + } + } + if content == "" { + content = title + } + return +} + +func (c *OfflinePushConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { + title, content, opts, err := c.getOfflinePushInfos(msg) + if err != nil { + return err + } + err = c.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) + if err != nil { + prommetrics.MsgOfflinePushFailedCounter.Inc() + return err + } + return nil +} diff --git a/internal/push/push.go b/internal/push/push.go index 1a04bbea26..850f91d22e 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,6 +2,7 @@ package push import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -17,12 +18,12 @@ type pushServer struct { disCov discovery.SvcDiscoveryRegistry offlinePusher offlinepush.OfflinePusher pushCh *ConsumerHandler + offlinePushCh *OfflinePushConsumerHandler } type Config struct { RpcConfig config.Push RedisConfig config.Redis - MongodbConfig config.Mongo KafkaConfig config.Kafka NotificationConfig config.Notification Share config.Share @@ -55,18 +56,30 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - database := controller.NewPushDatabase(cacheModel) - consumer, err := NewConsumerHandler(config, offlinePusher, rdb, client) + database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig) + + consumer, err := NewConsumerHandler(config, database, offlinePusher, rdb, client) + if err != nil { + return err + } + + offlinePushConsumer, err := NewOfflinePushConsumerHandler(config, offlinePusher) if err != nil { return err } + pbpush.RegisterPushMsgServiceServer(server, &pushServer{ database: database, disCov: client, offlinePusher: offlinePusher, pushCh: consumer, + offlinePushCh: offlinePushConsumer, }) + go consumer.pushConsumerGroup.RegisterHandleAndConsumer(ctx, consumer) + + go offlinePushConsumer.OfflinePushConsumerGroup.RegisterHandleAndConsumer(ctx, offlinePushConsumer) + return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 5d03599944..38d190ae5a 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -1,33 +1,20 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package push import ( "context" "encoding/json" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" - pbchat "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" @@ -46,6 +33,7 @@ type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + pushDatabase controller.PushDatabase onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache @@ -56,7 +44,7 @@ type ConsumerHandler struct { config *Config } -func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, +func NewConsumerHandler(config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) { var consumerHandler ConsumerHandler var err error @@ -65,6 +53,7 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) @@ -75,43 +64,42 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config + consumerHandler.pushDatabase = database consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) return &consumerHandler, nil } func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { - msgFromMQ := pbchat.PushMsgDataToMQ{} + msgFromMQ := pbpush.PushMsgReq{} if err := proto.Unmarshal(msg, &msgFromMQ); err != nil { log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg)) return } - pbData := &pbpush.PushMsgReq{ - MsgData: msgFromMQ.MsgData, - ConversationID: msgFromMQ.ConversationID, - } + sec := msgFromMQ.MsgData.SendTime / 1000 nowSec := timeutil.GetCurrentTimestampBySecond() if nowSec-sec > 10 { prommetrics.MsgLoneTimePushCounter.Inc() - log.ZWarn(ctx, "it’s been a while since the message was sent", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) + log.ZWarn(ctx, "it’s been a while since the message was sent", nil, "msg", msgFromMQ.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) } var err error + switch msgFromMQ.MsgData.SessionType { case constant.ReadGroupChatType: - err = c.Push2Group(ctx, pbData.MsgData.GroupID, pbData.MsgData) + err = c.Push2Group(ctx, msgFromMQ.MsgData.GroupID, msgFromMQ.MsgData) default: var pushUserIDList []string - isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) - if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) + isSenderSync := datautil.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) + if !isSenderSync || msgFromMQ.MsgData.SendID == msgFromMQ.MsgData.RecvID { + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID) } else { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID, msgFromMQ.MsgData.SendID) } - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) + err = c.Push2User(ctx, pushUserIDList, msgFromMQ.MsgData) } if err != nil { - log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) + log.ZWarn(ctx, "push failed", err, "msg", msgFromMQ.String()) } } @@ -246,28 +234,34 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s if err != nil { return err } + // Use offline push messaging if len(needOfflinePushUserIDs) > 0 { - var offlinePushUserIDs []string - err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) - if err != nil { - return err - } - - if len(offlinePushUserIDs) > 0 { - needOfflinePushUserIDs = offlinePushUserIDs - } + c.asyncOfflinePush(ctx, needOfflinePushUserIDs, msg) + } - err = c.offlinePushMsg(ctx, msg, needOfflinePushUserIDs) - if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg) - return nil - } + return nil +} +func (c *ConsumerHandler) asyncOfflinePush(ctx context.Context, needOfflinePushUserIDs []string, msg *sdkws.MsgData) { + var offlinePushUserIDs []string + err := c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) + if err != nil { + log.ZWarn(ctx, "webhookBeforeOfflinePush failed", err, "msg", msg) + return } - return nil + if len(offlinePushUserIDs) > 0 { + needOfflinePushUserIDs = offlinePushUserIDs + } + if err := c.pushDatabase.MsgToOfflinePushMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(msg.SendID, msg.RecvID), needOfflinePushUserIDs, msg); err != nil { + log.ZError(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs", + needOfflinePushUserIDs, "msg", msg) + prommetrics.SingleChatMsgProcessFailedCounter.Inc() + return + } } + func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index c9b8b1c245..ca22a697d2 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -37,7 +37,6 @@ func NewPushRpcCmd() *PushRpcCmd { ret.configMap = map[string]any{ OpenIMPushCfgFileName: &pushConfig.RpcConfig, RedisConfigFileName: &pushConfig.RedisConfig, - MongodbConfigFileName: &pushConfig.MongodbConfig, KafkaConfigFileName: &pushConfig.KafkaConfig, ShareFileName: &pushConfig.Share, NotificationFileName: &pushConfig.NotificationConfig, diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 8bd16178dd..80736d5cce 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -73,18 +73,21 @@ type Mongo struct { MaxRetry int `mapstructure:"maxRetry"` } type Kafka struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ProducerAck string `mapstructure:"producerAck"` - CompressType string `mapstructure:"compressType"` - Address []string `mapstructure:"address"` - ToRedisTopic string `mapstructure:"toRedisTopic"` - ToMongoTopic string `mapstructure:"toMongoTopic"` - ToPushTopic string `mapstructure:"toPushTopic"` - ToRedisGroupID string `mapstructure:"toRedisGroupID"` - ToMongoGroupID string `mapstructure:"toMongoGroupID"` - ToPushGroupID string `mapstructure:"toPushGroupID"` - Tls TLSConfig `mapstructure:"tls"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + ProducerAck string `mapstructure:"producerAck"` + CompressType string `mapstructure:"compressType"` + Address []string `mapstructure:"address"` + ToRedisTopic string `mapstructure:"toRedisTopic"` + ToMongoTopic string `mapstructure:"toMongoTopic"` + ToPushTopic string `mapstructure:"toPushTopic"` + ToOfflinePushTopic string `mapstructure:"toOfflinePushTopic"` + ToRedisGroupID string `mapstructure:"toRedisGroupID"` + ToMongoGroupID string `mapstructure:"toMongoGroupID"` + ToPushGroupID string `mapstructure:"toPushGroupID"` + ToOfflineGroupID string `mapstructure:"toOfflinePushGroupID"` + + Tls TLSConfig `mapstructure:"tls"` } type TLSConfig struct { EnableTLS bool `mapstructure:"enableTLS"` diff --git a/pkg/common/storage/controller/push.go b/pkg/common/storage/controller/push.go index 199a0ba678..91ef126fe5 100644 --- a/pkg/common/storage/controller/push.go +++ b/pkg/common/storage/controller/push.go @@ -17,21 +17,45 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" ) type PushDatabase interface { DelFcmToken(ctx context.Context, userID string, platformID int) error + MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error } type pushDataBase struct { - cache cache.ThirdCache + cache cache.ThirdCache + producerToOfflinePush *kafka.Producer } -func NewPushDatabase(cache cache.ThirdCache) PushDatabase { - return &pushDataBase{cache: cache} +func NewPushDatabase(cache cache.ThirdCache, kafkaConf *config.Kafka) PushDatabase { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil + } + producerToOfflinePush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToOfflinePushTopic) + if err != nil { + return nil + } + return &pushDataBase{ + cache: cache, + producerToOfflinePush: producerToOfflinePush, + } } func (p *pushDataBase) DelFcmToken(ctx context.Context, userID string, platformID int) error { return p.cache.DelFcmToken(ctx, userID, platformID) } + +func (p *pushDataBase) MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error { + _, _, err := p.producerToOfflinePush.SendMessage(ctx, key, &push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs}) + log.ZInfo(ctx, "message is push to offlinePush topic", "key", key, "userIDs", userIDs, "msg", msg2mq.String()) + return err +} diff --git a/scripts/create-topic.sh b/scripts/create-topic.sh index 206075fb83..bbc739287f 100755 --- a/scripts/create-topic.sh +++ b/scripts/create-topic.sh @@ -35,7 +35,7 @@ done echo "Kafka is ready. Creating topics..." -topics=("toRedis" "toMongo" "toPush") +topics=("toRedis" "toMongo" "toPush" "toOfflinePush") partitions=8 replicationFactor=1 diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5fa84ac36f..4f4c08c16a 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -18,6 +18,12 @@ import ( "context" "flag" "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/db/mongoutil" @@ -27,11 +33,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" - "io/ioutil" - "log" - "os" - "path/filepath" - "time" ) const maxRetry = 180 @@ -65,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error { } func CheckKafka(ctx context.Context, conf *config.Kafka) error { - return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic}) + return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic}) } func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) { From b7034496155e7c347616681bd574800d9ba96b24 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:48:34 +0800 Subject: [PATCH 89/91] feat: API supports gzip (#2609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao --- config/openim-api.yml | 3 +++ go.mod | 24 ++++++++++++++---------- go.sum | 30 ++++++++++++++++++++++++++++++ internal/api/router.go | 18 +++++++++++++++++- pkg/common/config/config.go | 5 +++-- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/config/openim-api.yml b/config/openim-api.yml index 9f53038d88..88560a155c 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -3,6 +3,9 @@ api: listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, must be consistent with the number of prometheus.ports ports: [ 10002 ] + # API compression level; 0: default compression, 1: best compression, 2: best speed, -1: no compression + compressionLevel: 0 + prometheus: # Whether to enable prometheus diff --git a/go.mod b/go.mod index d27ad28ea5..cf9cf710b9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( firebase.google.com/go v3.13.0+incompatible github.com/dtm-labs/rockscache v0.1.1 github.com/gin-gonic/gin v1.9.1 - github.com/go-playground/validator/v10 v10.18.0 + github.com/go-playground/validator/v10 v10.20.0 github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gorilla/websocket v1.5.1 @@ -20,7 +20,7 @@ require ( go.mongodb.org/mongo-driver v1.14.0 google.golang.org/api v0.165.0 google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -73,10 +73,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect github.com/aws/smithy-go v1.17.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -88,6 +91,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/gzip v1.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -117,7 +121,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lithammer/shortuuid v3.0.0+incompatible // indirect @@ -132,7 +136,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -169,9 +173,9 @@ require ( go.opentelemetry.io/otel/trace v1.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.3.0 // indirect + golang.org/x/arch v0.7.0 // indirect golang.org/x/image v0.15.0 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -187,10 +191,10 @@ require ( require ( github.com/go-playground/locales v0.14.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/spf13/cobra v1.8.0 - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 85ce53a62f..4771a0a7a0 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,10 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -77,6 +81,10 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583j github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= @@ -121,6 +129,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= +github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= @@ -146,6 +156,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= @@ -259,6 +271,9 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -290,6 +305,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -325,6 +342,8 @@ github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDIT github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -410,6 +429,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -458,6 +479,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -467,6 +490,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -495,6 +520,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= @@ -588,6 +615,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -608,6 +637,7 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/api/router.go b/internal/api/router.go index 1718505817..3817070b16 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "github.com/gin-contrib/gzip" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" @@ -22,6 +23,13 @@ import ( "github.com/openimsdk/tools/mw" ) +const ( + NoCompression = -1 + DefaultCompression = 0 + BestCompression = 1 + BestSpeed = 2 +) + func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { c.Next() @@ -54,7 +62,15 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - + switch config.API.Api.CompressionLevel { + case NoCompression: + case DefaultCompression: + r.Use(gzip.Gzip(gzip.DefaultCompression)) + case BestCompression: + r.Use(gzip.Gzip(gzip.BestCompression)) + case BestSpeed: + r.Use(gzip.Gzip(gzip.BestSpeed)) + } r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 80736d5cce..932ec9c25c 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -100,8 +100,9 @@ type TLSConfig struct { type API struct { Api struct { - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + ListenIP string `mapstructure:"listenIP"` + Ports []int `mapstructure:"ports"` + CompressionLevel int `mapstructure:"compressionLevel"` } `mapstructure:"api"` Prometheus struct { Enable bool `mapstructure:"enable"` From 0276c7df60e89df39b947fe77d13e821318402e1 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:38:17 +0800 Subject: [PATCH 90/91] Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee --- config/grafana-template/Demo.json | 3696 +++++++++-------- config/openim-api.yml | 2 +- config/openim-msggateway.yml | 7 +- config/openim-msgtransfer.yml | 2 +- config/openim-push.yml | 13 +- config/openim-rpc-auth.yml | 5 +- config/openim-rpc-conversation.yml | 4 +- config/openim-rpc-friend.yml | 4 +- config/openim-rpc-group.yml | 6 +- config/openim-rpc-msg.yml | 7 +- config/openim-rpc-third.yml | 6 +- config/openim-rpc-user.yml | 8 +- config/prometheus.yml | 25 +- docker-compose.yml | 1 - go.mod | 4 +- go.sum | 8 +- internal/msggateway/init.go | 2 +- internal/msggateway/ws_server.go | 6 +- internal/msgtransfer/init.go | 2 +- .../msgtransfer/online_history_msg_handler.go | 16 +- .../online_msg_to_mongo_handler.go | 2 +- internal/push/offlinepush/dummy/push.go | 2 +- internal/push/offlinepush/getui/push.go | 2 - internal/push/offlinepush_handler.go | 8 +- internal/push/onlinepusher.go | 4 +- internal/push/push_handler.go | 65 +- internal/rpc/msg/clear.go | 2 +- internal/rpc/user/online.go | 21 + internal/tools/cron_task.go | 14 +- pkg/common/config/config.go | 1 + pkg/common/startrpc/start.go | 6 +- pkg/common/storage/cache/cachekey/online.go | 9 +- pkg/common/storage/cache/online.go | 1 + pkg/common/storage/cache/redis/batch.go | 6 +- .../storage/cache/redis/batch_handler.go | 6 + pkg/common/storage/cache/redis/online.go | 32 + pkg/localcache/lru/lru.go | 2 + pkg/localcache/lru/lru_expiration.go | 11 + pkg/localcache/lru/lru_lazy.go | 70 +- pkg/localcache/lru/lru_slot.go | 27 + pkg/rpccache/online.go | 268 +- pkg/rpcclient/user.go | 13 + start-config.yml | 4 +- 43 files changed, 2467 insertions(+), 1933 deletions(-) diff --git a/config/grafana-template/Demo.json b/config/grafana-template/Demo.json index c4668917f4..ea17d2c0ab 100644 --- a/config/grafana-template/Demo.json +++ b/config/grafana-template/Demo.json @@ -54,7 +54,7 @@ "liveNow": false, "panels": [ { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -62,1120 +62,1251 @@ "y": 0 }, "id": 35, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Is the service up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Is the service up.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 9, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bool_on_off" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 6, - "y": 1 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "lineWidth": 2, + "pointSize": 9, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "up", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "$legendName", - "range": true, - "refId": "A" - } - ], - "title": "UP", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of online users and login users within the time frame.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "online users" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#37bbff", - "mode": "fixed", - "seriesBy": "last" - } - } - ] - } - ] + "editorMode": "code", + "exemplar": false, + "expr": "up", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "UP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of online users and login users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 37, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "online_user_num", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "online users", - "range": true, - "refId": "A" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(user_login_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "login num", - "range": true, - "refId": "B" + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Login Information", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of register users within the time frame.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "none" - }, - "overrides": [ { - "matcher": { - "id": "byName", - "options": "register users" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7437ff", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 59, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "online users" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#37bbff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "user_register_total", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "register users", - "range": true, - "refId": "A" - } - ], - "title": "Register num", - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "online_user_num", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "online users", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of chat msg success.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "expr": "increase(user_login_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "login num", + "range": true, + "refId": "B" + } + ], + "title": "Login Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of register users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "none" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "register users" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7437ff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 23 + "editorMode": "code", + "exemplar": false, + "expr": "user_register_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "register users", + "range": true, + "refId": "A" + } + ], + "title": "Register num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg success.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 38, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(single_chat_msg_process_success_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "single msgs", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "increase(group_chat_msg_process_success_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "group msgs", - "range": true, - "refId": "B" - } - ], - "title": "Chat Msg Success Num", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of chat msg failed .", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Success Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg failed .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "single msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#ff00dc", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "green", + "value": null }, { - "matcher": { - "id": "byName", - "options": "group msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0cffef", - "mode": "fixed" - } - } - ] + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 39, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "single msgs" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#ff00dc", + "mode": "fixed", + "seriesBy": "last" + } + } + ] }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(single_chat_msg_process_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "single msgs", - "range": true, - "refId": "A" + { + "matcher": { + "id": "byName", + "options": "group msgs" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(group_chat_msg_process_failed_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "group msgs", - "range": true, - "refId": "B" - } - ], - "title": "Chat Msg Failed Num", - "type": "timeseries" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0cffef", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of msg failed offline pushed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failed msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed", - "seriesBy": "last" - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 4, - "y": 33 - }, - "id": 42, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_offline_push_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "failed msgs", - "range": true, - "refId": "A" - } - ], - "title": "Msg Offline Push Failed Num", - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of failed set seq.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "editorMode": "code", + "expr": "increase(group_chat_msg_process_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of msg failed offline pushed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "failed msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 11, - "w": 6, - "x": 14, - "y": 33 - }, - "id": 43, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(seq_set_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "failed addr: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Seq Set Failed Num", - "type": "timeseries" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of successfully inserted messages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_offline_push_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Msg Offline Push Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed set seq.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed", + "seriesBy": "last" } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(seq_set_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Seq Set Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of messages that take a long time to send.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 44 + "editorMode": "code", + "exemplar": false, + "expr": "msg_long_time_push_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Long Time Send Msg Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of successfully inserted messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 44, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_insert_redis_success_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "redis: {{instance}}", - "range": true, - "refId": "A" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Success Insert Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed insertion messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(msg_insert_mongo_success_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "mongo: {{instance}}", - "range": true, - "refId": "B" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Msg Success Insert Num", - "type": "timeseries" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of failed insertion messages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 45, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_insert_redis_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "redis: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(msg_insert_mongo_failed_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "mongo: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Msg Failed Insert Num", - "type": "timeseries" + "editorMode": "code", + "expr": "increase(msg_insert_mongo_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" } ], - "title": "Server", - "type": "row" + "title": "Msg Failed Insert Num", + "type": "timeseries" }, { "collapsed": true, @@ -1183,7 +1314,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 54 }, "id": 22, "panels": [ @@ -1973,7 +2104,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 55 }, "id": 28, "panels": [ @@ -2827,7 +2958,7 @@ "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 56 }, "id": 25, "panels": [ @@ -3377,849 +3508,848 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 57 }, "id": 6, - "panels": [], - "title": "Process", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 5 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process virtual memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "unit": "bytes" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Virtual Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process virtual memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 27 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Virtual Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Resident Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" } ], - "title": "Resident Memory bytes", - "type": "timeseries" + "title": "Process", + "type": "row" }, { "collapsed": true, @@ -4227,7 +4357,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 58 }, "id": 3, "panels": [ @@ -5441,6 +5571,6 @@ "timezone": "", "title": "Demo", "uid": "a506d250-b606-4702-86a7-ac6aa1d069a1", - "version": 23, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/config/openim-api.yml b/config/openim-api.yml index 88560a155c..4c38e1005b 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -11,6 +11,6 @@ prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must match the number of api.ports - ports: [ 20502 ] + ports: [ 12002 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 63332740ff..428f3ba476 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -2,13 +2,13 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10140 ] + ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20640 ] + ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 @@ -25,6 +25,3 @@ longConnSvr: # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLoginPolicy: 1 - - - diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index e71a218edf..753ac10bc0 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -3,4 +3,4 @@ prometheus: enable: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # Because four instances have been launched, four ports need to be specified - ports: [ 20600, 20601, 20602, 20603 ] + ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 70aa5997f5..6df2a62b7f 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -1,16 +1,16 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170, 10171, 10172, 10173 ] + ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20670, 20671, 20672, 20673 ] + ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185 ] maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. @@ -38,9 +38,4 @@ iosPush: badgeCount: true production: false - - - - - - +fullUserCache: true diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index c55c745b6c..496803e43b 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -4,15 +4,14 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10160 ] + ports: [ 10200 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20660 ] + ports: [ 12200 ] tokenPolicy: # Token validity period, in days expire: 90 - diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 00c9c5aab9..3581d7e19e 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -4,10 +4,10 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10180 ] + ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20680 ] + ports: [ 12220 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index afac3c5db9..3022c09f32 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -4,10 +4,10 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10120 ] + ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20620 ] + ports: [ 12240 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index d0243b5fbb..9a634d12ff 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -4,13 +4,13 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10150 ] + ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20650 ] + ports: [ 12260 ] -enableHistoryForNewMembers: true \ No newline at end of file +enableHistoryForNewMembers: true diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 15840c7f3b..82d6e2f539 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -4,17 +4,14 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10130 ] + ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20630 ] + ports: [ 12280 ] # Does sending messages require friend verification friendVerify: false - - - diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 512ca391fa..d8f2d427f2 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -4,13 +4,13 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10190 ] + ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20690 ] + ports: [ 12300 ] object: @@ -37,4 +37,4 @@ object: accessKeyID: accessKeySecret: sessionToken: - publicRead: false \ No newline at end of file + publicRead: false diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 1958bbc6ab..798105472c 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -4,14 +4,10 @@ rpc: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports - ports: [ 10110 ] + ports: [ 10320 ] prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports - ports: [ 20610 ] - - - - + ports: [ 12320 ] diff --git a/config/prometheus.yml b/config/prometheus.yml index 627cf94114..4f0f7e32c0 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -28,56 +28,59 @@ scrape_configs: - targets: [ internal_ip:20500 ] - job_name: openimserver-openim-api static_configs: - - targets: [ internal_ip:20502 ] + - targets: [ internal_ip:12002 ] labels: namespace: default - job_name: openimserver-openim-msggateway static_configs: - - targets: [ internal_ip:20640 ] + - targets: [ internal_ip:12140 ] +# - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] labels: namespace: default - job_name: openimserver-openim-msgtransfer static_configs: - - targets: [ internal_ip:20600, internal_ip:20601, internal_ip:20602, internal_ip:20603 ] + - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] +# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] labels: namespace: default - job_name: openimserver-openim-push static_configs: - - targets: [ internal_ip:20670, internal_ip:20671, internal_ip:20672, internal_ip:20673] + - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12181, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185 ] labels: namespace: default - job_name: openimserver-openim-rpc-auth static_configs: - - targets: [ internal_ip:20600 ] + - targets: [ internal_ip:12200 ] labels: namespace: default - job_name: openimserver-openim-rpc-conversation static_configs: - - targets: [ internal_ip:20680 ] + - targets: [ internal_ip:12220 ] labels: namespace: default - job_name: openimserver-openim-rpc-friend static_configs: - - targets: [ internal_ip:20620 ] + - targets: [ internal_ip:12240 ] labels: namespace: default - job_name: openimserver-openim-rpc-group static_configs: - - targets: [ internal_ip:20650 ] + - targets: [ internal_ip:12260 ] labels: namespace: default - job_name: openimserver-openim-rpc-msg static_configs: - - targets: [ internal_ip:20630 ] + - targets: [ internal_ip:12280 ] labels: namespace: default - job_name: openimserver-openim-rpc-third static_configs: - - targets: [ internal_ip:20690 ] + - targets: [ internal_ip:12300 ] labels: namespace: default - job_name: openimserver-openim-rpc-user static_configs: - - targets: [ internal_ip:20610 ] + - targets: [ internal_ip:12320 ] labels: namespace: default \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 49c44d3d9a..512f951dbd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -186,4 +186,3 @@ services: # networks: # - openim - diff --git a/go.mod b/go.mod index cf9cf710b9..cec96b5882 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.17 - github.com/openimsdk/tools v0.0.50-alpha.11 + github.com/openimsdk/protocol v0.0.72-alpha.18 + github.com/openimsdk/tools v0.0.50-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 4771a0a7a0..be7b7b3ad5 100644 --- a/go.sum +++ b/go.sum @@ -336,10 +336,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.17 h1:kB7eyjJHdkc8lpSlLIHskHzbodxkIG4eaK908iQLVdI= -github.com/openimsdk/protocol v0.0.72-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.50-alpha.11 h1:ClhkRjUVJWbmOiQ14G6do/ES1a6ZueDITv40Apwq/Tc= -github.com/openimsdk/tools v0.0.50-alpha.11/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/protocol v0.0.72-alpha.18 h1:EytTtgZuXMG1cgTlJryqXXSO1J3t3wrLIn3Os2PRBEE= +github.com/openimsdk/protocol v0.0.72-alpha.18/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo= +github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 44e79e4122..50da060976 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -58,7 +58,7 @@ func Start(ctx context.Context, index int, conf *Config) error { ) hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { - longServer.online = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, longServer.subscriberUserOnlineStatusChanges) + longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) return nil }) diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 9b18ade7dc..81392897bf 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -265,7 +265,7 @@ func (ws *WsServer) registerClient(client *Client) { if clientOK { ws.clients.Set(client.UserID, client) // There is already a connection to the platform - log.ZInfo(client.ctx, "repeat login", "userID", client.UserID, "platformID", + log.ZDebug(client.ctx, "repeat login", "userID", client.UserID, "platformID", client.PlatformID, "old remote addr", getRemoteAdders(oldClients)) ws.onlineUserConnNum.Add(1) } else { @@ -293,7 +293,7 @@ func (ws *WsServer) registerClient(client *Client) { wg.Wait() - log.ZInfo( + log.ZDebug( client.ctx, "user online", "online user Num", @@ -360,7 +360,7 @@ func (ws *WsServer) unregisterClient(client *Client) { ws.onlineUserConnNum.Add(-1) ws.subscription.DelClient(client) //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) - log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", + log.ZDebug(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), ) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index a1c8b26b6c..7dc2ebeea0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -111,7 +111,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - + msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 6b924b05b9..b0078649cb 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -238,6 +238,7 @@ func (och *OnlineHistoryRedisConsumerHandler) categorizeMessageLists(totalMsgs [ } func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key, conversationID string, storageList, notStorageList []*ContextMsg) { + log.ZInfo(ctx, "handle storage msg") for _, storageMsg := range storageList { log.ZDebug(ctx, "handle storage msg", "msg", storageMsg.message.String()) } @@ -254,16 +255,20 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } + log.ZInfo(ctx, "BatchInsertChat2Cache end") + if isNewConversation { switch msg.SessionType { case constant.ReadGroupChatType: - log.ZInfo(ctx, "group chat first create conversation", "conversationID", + log.ZDebug(ctx, "group chat first create conversation", "conversationID", conversationID) userIDs, err := och.groupRpcClient.GetGroupMemberIDs(ctx, msg.GroupID) if err != nil { log.ZWarn(ctx, "get group member ids error", err, "conversationID", conversationID) } else { + log.ZInfo(ctx, "GetGroupMemberIDs end") + if err := och.conversationRpcClient.GroupChatFirstCreateConversation(ctx, msg.GroupID, userIDs); err != nil { log.ZWarn(ctx, "single chat first create conversation error", err, @@ -282,13 +287,16 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } } - log.ZDebug(ctx, "success incr to next topic") + log.ZInfo(ctx, "success incr to next topic") err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) } + log.ZInfo(ctx, "MsgToMongoMQ end") + och.toPushTopic(ctx, key, conversationID, storageList) + log.ZInfo(ctx, "toPushTopic end") } } @@ -319,7 +327,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) - och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) + _, _, _ = och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) } } @@ -344,7 +352,7 @@ func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSess func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - log.ZInfo(context.Background(), "online new session msg come", "highWaterMarkOffset", + log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) och.redisMessageBatches.OnComplete = func(lastMessage *sarama.ConsumerMessage, totalCount int) { session.MarkMessage(lastMessage, "") diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index ef6f6ac7d9..82002c26b9 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -57,7 +57,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont log.ZError(ctx, "msgFromMQ.MsgData is empty", nil, "cMsg", cMsg) return } - log.ZInfo(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) + log.ZDebug(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) err = mc.msgTransferDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) if err != nil { log.ZError( diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 5698b72947..09831cabfa 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -28,6 +28,6 @@ type Dummy struct { } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { - log.ZInfo(ctx, "dummy push") + log.ZDebug(ctx, "dummy push") return nil } diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index 674d081160..e266f9c464 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -23,7 +23,6 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/errs" @@ -100,7 +99,6 @@ func (g *Client) Push(ctx context.Context, userIDs []string, title, content stri if err = g.batchPush(ctx, token, userIDs[i:end], pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) } - } if err = g.batchPush(ctx, token, userIDs, pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go index e97e0e4db8..bf69aed3e2 100644 --- a/internal/push/offlinepush_handler.go +++ b/internal/push/offlinepush_handler.go @@ -63,7 +63,7 @@ func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, } } -func (c *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, content string, opts *options.Opts, err error) { +func (o *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, content string, opts *options.Opts, err error) { type AtTextElem struct { Text string `json:"text,omitempty"` AtUserList []string `json:"atUserList,omitempty"` @@ -108,12 +108,12 @@ func (c *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (ti return } -func (c *OfflinePushConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { - title, content, opts, err := c.getOfflinePushInfos(msg) +func (o *OfflinePushConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { + title, content, opts, err := o.getOfflinePushInfos(msg) if err != nil { return err } - err = c.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) + err = o.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) if err != nil { prommetrics.MsgOfflinePushFailedCounter.Inc() return err diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index d0c65e06bf..9521a84a07 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -27,12 +27,12 @@ func newEmptyOnlinePusher() *emptyOnlinePusher { func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - log.ZWarn(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) return nil, nil } func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { - log.ZWarn(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 38d190ae5a..4ecf20de52 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -27,6 +27,9 @@ import ( "github.com/openimsdk/tools/utils/timeutil" "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" + "math/rand" + "strconv" + "time" ) type ConsumerHandler struct { @@ -55,6 +58,7 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin } userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) @@ -65,7 +69,10 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config consumerHandler.pushDatabase = database - consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) + consumerHandler.onlineCache, err = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) + if err != nil { + return nil, err + } return &consumerHandler, nil } @@ -108,6 +115,14 @@ func (*ConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } func (*ConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + c.onlineCache.Lock.Lock() + for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver { + c.onlineCache.Cond.Wait() + } + c.onlineCache.Lock.Unlock() + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + log.ZInfo(ctx, "begin consume messages") + for msg := range claim.Messages() { ctx := c.pushConsumerGroup.GetContextFromMsg(msg) c.handleMs2PsChat(ctx, msg.Value) @@ -118,20 +133,27 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "msg", msg.String(), "time cost", t) + }(time.Now()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } + log.ZInfo(ctx, "webhookBeforeOnlinePush end") + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } - log.ZDebug(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) + log.ZInfo(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) if !c.shouldPushOffline(ctx, msg) { return nil } + log.ZInfo(ctx, "shouldPushOffline end") for _, v := range wsResults { //message sender do not need offline push @@ -150,7 +172,7 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * offlinePushUserID, msg, nil); err != nil { return err } - + log.ZInfo(ctx, "webhookBeforeOfflinePush end") err = c.offlinePushMsg(ctx, msg, offlinePushUserID) if err != nil { log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) @@ -172,21 +194,11 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat } func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { - var ( - onlineUserIDs []string - offlineUserIDs []string - ) - for _, userID := range pushToUserIDs { - online, err := c.onlineCache.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } else { - offlineUserIDs = append(offlineUserIDs, userID) - } + onlineUserIDs, offlineUserIDs, err := c.onlineCache.GetUsersOnline(ctx, pushToUserIDs) + if err != nil { + return nil, err } + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { @@ -205,35 +217,42 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. } func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg end", "msg", msg.String(), "groupID", groupID, "time cost", t) + }(time.Now()) var pushToUserIDs []string if err = c.webhookBeforeGroupOnlinePush(ctx, &c.config.WebhooksConfig.BeforeGroupOnlinePush, groupID, msg, &pushToUserIDs); err != nil { return err } + log.ZInfo(ctx, "webhookBeforeGroupOnlinePush end") err = c.groupMessagesHandler(ctx, groupID, &pushToUserIDs, msg) if err != nil { return err } + log.ZInfo(ctx, "groupMessagesHandler end") wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } - log.ZDebug(ctx, "group push result", "result", wsResults, "msg", msg) + log.ZInfo(ctx, "group push result", "result", wsResults, "msg", msg) if !c.shouldPushOffline(ctx, msg) { return nil } needOfflinePushUserIDs := c.onlinePusher.GetOnlinePushFailedUserIDs(ctx, msg, wsResults, &pushToUserIDs) - + log.ZInfo(ctx, "GetOnlinePushFailedUserIDs end") //filter some user, like don not disturb or don't need offline push etc. needOfflinePushUserIDs, err = c.filterGroupMessageOfflinePush(ctx, groupID, msg, needOfflinePushUserIDs) if err != nil { return err } + log.ZInfo(ctx, "filterGroupMessageOfflinePush end") // Use offline push messaging if len(needOfflinePushUserIDs) > 0 { @@ -295,7 +314,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri if unmarshalNotificationElem(msg.Content, &tips) != nil { return err } - log.ZInfo(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) + log.ZDebug(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) if len(c.config.Share.IMAdminUserID) > 0 { ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) } @@ -379,6 +398,7 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten } return } + func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) maxSeq, err := c.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) @@ -387,6 +407,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, } return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } + func unmarshalNotificationElem(bytes []byte, t any) error { var notification sdkws.NotificationElem if err := json.Unmarshal(bytes, ¬ification); err != nil { diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 4ffa1f43ee..c5bd36b445 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -67,7 +67,7 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, err } - log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) return &msg.ClearMsgResp{}, nil } diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go index 99b272006f..4e7823306f 100644 --- a/internal/rpc/user/online.go +++ b/internal/rpc/user/online.go @@ -2,6 +2,8 @@ package user import ( "context" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/protocol/constant" pbuser "github.com/openimsdk/protocol/user" ) @@ -80,3 +82,22 @@ func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUse } return &pbuser.SetUserOnlineStatusResp{}, nil } + +func (s *userServer) GetAllOnlineUsers(ctx context.Context, req *pbuser.GetAllOnlineUsersReq) (*pbuser.GetAllOnlineUsersResp, error) { + resMap, nextCursor, err := s.online.GetAllOnlineUsers(ctx, req.Cursor) + if err != nil { + return nil, err + } + resp := &pbuser.GetAllOnlineUsersResp{ + StatusList: make([]*pbuser.OnlineStatus, 0, len(resMap)), + NextCursor: nextCursor, + } + for userID, plats := range resMap { + resp.StatusList = append(resp.StatusList, &pbuser.OnlineStatus{ + UserID: userID, + Status: int32(datautil.If(len(plats) > 0, constant.Online, constant.Offline)), + PlatformIDs: plats, + }) + } + return resp, nil +} diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 337272d69d..dbb4e34f61 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -79,13 +79,13 @@ func Start(ctx context.Context, config *CronTaskConfig) error { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + log.ZDebug(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } - log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { return errs.Wrap(err) @@ -95,7 +95,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { msgDestructFunc := func() { now := time.Now() ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZInfo(ctx, "msg destruct cron start", "now", now) + log.ZDebug(ctx, "msg destruct cron start", "now", now) conversations, err := conversationClient.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) if err != nil { @@ -108,7 +108,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return } } - log.ZInfo(ctx, "msg destruct cron task completed", "cont", time.Since(now)) + log.ZDebug(ctx, "msg destruct cron task completed", "cont", time.Since(now)) } if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, msgDestructFunc); err != nil { return errs.Wrap(err) @@ -119,18 +119,18 @@ func Start(ctx context.Context, config *CronTaskConfig) error { // now := time.Now() // deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) // ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - // log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + // log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) // if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { // log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) // return // } - // log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + // log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) // } // if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { // return errs.Wrap(err) // } - log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) + log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 932ec9c25c..59919208be 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -224,6 +224,7 @@ type Push struct { BadgeCount bool `mapstructure:"badgeCount"` Production bool `mapstructure:"production"` } `mapstructure:"iosPush"` + FullUserCache bool `mapstructure:"fullUserCache"` } type Auth struct { diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 4091a5f6e6..85a6c3d512 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -54,15 +54,11 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort, "prometheusPorts", prometheusConfig.Ports) rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) + listener, err := net.Listen( "tcp", rpcTcpAddr, ) - if err != nil { - return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } - - defer listener.Close() client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { return err diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go index 164e5f2f46..40f09cb5ae 100644 --- a/pkg/common/storage/cache/cachekey/online.go +++ b/pkg/common/storage/cache/cachekey/online.go @@ -1,6 +1,9 @@ package cachekey -import "time" +import ( + "strings" + "time" +) const ( OnlineKey = "ONLINE:" @@ -11,3 +14,7 @@ const ( func GetOnlineKey(userID string) string { return OnlineKey + userID } + +func GetOnlineKeyUserID(key string) string { + return strings.TrimPrefix(key, OnlineKey) +} diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go index 7669c8a118..d21ae616a6 100644 --- a/pkg/common/storage/cache/online.go +++ b/pkg/common/storage/cache/online.go @@ -5,4 +5,5 @@ import "context" type OnlineCache interface { GetOnline(ctx context.Context, userID string) ([]int32, error) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error + GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) } diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 4d65c59298..1810ac9939 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "golang.org/x/sync/singleflight" @@ -65,6 +66,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } bs, err := json.Marshal(value) if err != nil { + log.ZError(ctx, "marshal failed", err) return nil, err } cacheIndex[index] = string(bs) @@ -72,7 +74,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac return cacheIndex, nil }) if err != nil { - return nil, err + return nil, errs.WrapMsg(err, "FetchBatch2 failed") } for index, data := range indexCache { if data == "" { @@ -80,7 +82,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } var value V if err := json.Unmarshal([]byte(data), &value); err != nil { - return nil, err + return nil, errs.WrapMsg(err, "Unmarshal failed") } if cb, ok := any(&value).(BatchCacheCallback[K]); ok { cb.BatchCache(keyId[keys[index]]) diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 1fbd664a32..420ebdf777 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -28,6 +28,10 @@ import ( "time" ) +const ( + rocksCacheTimeout = 11 * time.Second +) + // BatchDeleterRedis is a concrete implementation of the BatchDeleter interface based on Redis and RocksCache. type BatchDeleterRedis struct { redisClient redis.UniversalClient @@ -106,6 +110,8 @@ func (c *BatchDeleterRedis) AddKeys(keys ...string) { // GetRocksCacheOptions returns the default configuration options for RocksCache. func GetRocksCacheOptions() *rockscache.Options { opts := rockscache.NewDefaultOptions() + opts.LockExpire = rocksCacheTimeout + opts.WaitReplicasTimeout = rocksCacheTimeout opts.StrongConsistency = true opts.RandomExpireAdjustment = 0.2 diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index ee1db7e23b..b6c90264e1 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -2,8 +2,10 @@ package redis import ( "context" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -49,6 +51,36 @@ func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, err return platformIDs, nil } +func (s *userOnline) GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) { + result := make(map[string][]int32) + + keys, nextCursor, err := s.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", cachekey.OnlineKey), constant.ParamMaxLength).Result() + if err != nil { + return nil, 0, err + } + + for _, key := range keys { + userID := cachekey.GetOnlineKeyUserID(key) + strValues, err := s.rdb.ZRange(ctx, key, 0, -1).Result() + if err != nil { + return nil, 0, err + } + + values := make([]int32, 0, len(strValues)) + for _, value := range strValues { + intValue, err := strconv.Atoi(value) + if err != nil { + return nil, 0, errs.Wrap(err) + } + values = append(values, int32(intValue)) + } + + result[userID] = values + } + + return result, nextCursor, nil +} + func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { script := ` local key = KEYS[1] diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 2fedffc48b..726535c48c 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,7 +20,9 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + Set(key K, value V) SetHas(key K, value V) bool + GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index d27e670574..df6bacbf43 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -51,6 +51,11 @@ type ExpirationLRU[K comparable, V any] struct { target Target } +func (x *ExpirationLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + //TODO implement me + panic("implement me") +} + func (x *ExpirationLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { x.lock.Lock() v, ok := x.core.Get(key) @@ -99,5 +104,11 @@ func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { return false } +func (x *ExpirationLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &expirationLruItem[V]{value: value}) +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index e935c687c4..e7f7b8bd51 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,18 +88,76 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } -//func (x *LayLRU[K, V]) Set(key K, value V) { -// x.lock.Lock() -// x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) -// x.lock.Unlock() -//} -// +func (x *LayLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + err error + once sync.Once + ) + + x.lock.Lock() + res := make(map[K]V) + queries := make([]K, 0) + setVs := make(map[K]*layLruItem[V]) + for _, key := range keys { + v, ok := x.core.Get(key) + if ok { + x.lock.Unlock() + v.lock.Lock() + expires, value, err1 := v.expires, v.value, v.err + if expires != 0 && expires > time.Now().UnixMilli() { + v.lock.Unlock() + x.target.IncrGetHit() + res[key] = value + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + continue + } + } + queries = append(queries, key) + x.lock.Unlock() + } + values, err1 := fetch(queries) + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + for key, val := range values { + v := &layLruItem[V]{} + v.value = val + + if err == nil { + v.expires = time.Now().Add(x.successTTL).UnixMilli() + x.target.IncrGetSuccess() + } else { + v.expires = time.Now().Add(x.failedTTL).UnixMilli() + x.target.IncrGetFailed() + } + setVs[key] = v + x.lock.Lock() + x.core.Add(key, v) + x.lock.Unlock() + res[key] = val + } + + return res, err +} + //func (x *LayLRU[K, V]) Has(key K) bool { // x.lock.Lock() // defer x.lock.Unlock() // return x.core.Contains(key) //} +func (x *LayLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +} + func (x *LayLRU[K, V]) SetHas(key K, value V) bool { x.lock.Lock() defer x.lock.Unlock() diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index 4538ca20e4..077219b75f 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -32,6 +32,29 @@ type slotLRU[K comparable, V any] struct { hash func(k K) uint64 } +func (x *slotLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + slotKeys = make(map[uint64][]K) + vs = make(map[K]V) + ) + + for _, k := range keys { + index := x.getIndex(k) + slotKeys[index] = append(slotKeys[index], k) + } + + for k, v := range slotKeys { + batches, err := x.slots[k].GetBatch(v, fetch) + if err != nil { + return nil, err + } + for key, value := range batches { + vs[key] = value + } + } + return vs, nil +} + func (x *slotLRU[K, V]) getIndex(k K) uint64 { return x.hash(k) % x.n } @@ -40,6 +63,10 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) Set(key K, value V) { + x.slots[x.getIndex(key)].Set(key, value) +} + func (x *slotLRU[K, V]) SetHas(key K, value V) bool { return x.slots[x.getIndex(key)].SetHas(key, value) } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 2ffa1f1577..a02a0662d4 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -2,60 +2,197 @@ package rpccache import ( "context" + "fmt" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" + "math/rand" + "strconv" + "sync" + "sync/atomic" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/db/cacheutil" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/redis/go-redis/v9" - "math/rand" - "strconv" - "time" ) -func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) *OnlineCache { +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { + l := &sync.Mutex{} x := &OnlineCache{ - user: user, - group: group, - local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + user: user, + group: group, + fullUserCache: fullUserCache, + Lock: l, + Cond: sync.NewCond(l), + } + + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + + switch x.fullUserCache { + case true: + log.ZDebug(ctx, "fullUserCache is true") + x.mapCache = cacheutil.NewCache[string, []int32]() + go func() { + if err := x.initUsersOnlineStatus(ctx); err != nil { + log.ZError(ctx, "initUsersOnlineStatus failed", err) + } + }() + case false: + log.ZDebug(ctx, "fullUserCache is false") + x.lruCache = lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) - }), + }) + x.CurrentPhase.Store(DoSubscribeOver) + x.Cond.Broadcast() } + go func() { - ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) - for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + x.doSubscribe(ctx, rdb, fn) + }() + return x, nil +} + +const ( + Begin uint32 = iota + DoOnlineStatusOver + DoSubscribeOver +) + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + + // fullUserCache if enabled, caches the online status of all users using mapCache; + // otherwise, only a portion of users' online statuses (regardless of whether they are online) will be cached using lruCache. + fullUserCache bool + + lruCache lru.LRU[string, []int32] + mapCache *cacheutil.Cache[string, []int32] + + Lock *sync.Mutex + Cond *sync.Cond + CurrentPhase atomic.Uint32 +} + +func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { + log.ZDebug(ctx, "init users online status begin") + + var ( + totalSet atomic.Int64 + maxTries = 5 + retryInterval = time.Second * 5 + + resp *user.GetAllOnlineUsersResp + ) + + defer func(t time.Time) { + log.ZInfo(ctx, "init users online status end", "cost", time.Since(t), "totalSet", totalSet.Load()) + o.CurrentPhase.Store(DoOnlineStatusOver) + o.Cond.Broadcast() + }(time.Now()) + + retryOperation := func(operation func() error, operationName string) error { + for i := 0; i < maxTries; i++ { + if err = operation(); err != nil { + log.ZWarn(ctx, fmt.Sprintf("initUsersOnlineStatus: %s failed", operationName), err) + time.Sleep(retryInterval) + } else { + return nil + } + } + return err + } + + cursor := uint64(0) + for resp == nil || resp.NextCursor != 0 { + if err = retryOperation(func() error { + resp, err = o.user.GetAllOnlineUsers(ctx, cursor) if err != nil { - log.ZError(ctx, "OnlineCache setUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) - continue + return err + } + + for _, u := range resp.StatusList { + if u.Status == constant.Online { + o.setUserOnline(u.UserID, u.PlatformIDs) + } + totalSet.Add(1) } - storageCache := x.setUserOnline(userID, platformIDs) - log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + cursor = resp.NextCursor + return nil + }, "getAllOnlineUsers"); err != nil { + return err + } + } + + return nil +} + +func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) { + o.Lock.Lock() + ch := rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() + for o.CurrentPhase.Load() < DoOnlineStatusOver { + o.Cond.Wait() + } + o.Lock.Unlock() + log.ZInfo(ctx, "begin doSubscribe") + + doMessage := func(message *redis.Message) { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache setHasUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + return + } + log.ZDebug(ctx, fmt.Sprintf("get subscribe %s message", cachekey.OnlineChannel), "useID", userID, "platformIDs", platformIDs) + switch o.fullUserCache { + case true: + if len(platformIDs) == 0 { + // offline + o.mapCache.Delete(userID) + } else { + o.mapCache.Store(userID, platformIDs) + } + case false: + storageCache := o.setHasUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setHasUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) if fn != nil { fn(ctx, userID, platformIDs) } } - }() - return x -} + } -type OnlineCache struct { - user rpcclient.UserRpcClient - group *GroupLocalCache - local lru.LRU[string, []int32] + if o.CurrentPhase.Load() == DoOnlineStatusOver { + for done := false; !done; { + select { + case message := <-ch: + doMessage(message) + default: + o.CurrentPhase.Store(DoSubscribeOver) + o.Cond.Broadcast() + done = true + } + } + } + + for message := range ch { + doMessage(message) + } } func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - platformIDs, err := o.local.Get(userID, func() ([]int32, error) { + platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) if err != nil { log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) return nil, err } - log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + //log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) return platformIDs, nil } @@ -69,6 +206,16 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } +// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { +// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) +// if err != nil { +// return nil, err +// } +// tmp := make([]int32, len(platformIDs)) +// copy(tmp, platformIDs) +// return platformIDs, nil +// } + func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { @@ -77,10 +224,68 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } +func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { + platformIDsMap := make(map[string][]int32) + + usersStatus, err := o.user.GetUsersOnlinePlatform(ctx, missingUsers) + if err != nil { + return nil, err + } + + for _, u := range usersStatus { + platformIDsMap[u.UserID] = u.PlatformIDs + } + + return platformIDsMap, nil + }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userIDs) + return nil, err + } + return platformIDsMap, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { + t := time.Now() + + var ( + onlineUserIDs = make([]string, 0, len(userIDs)) + offlineUserIDs = make([]string, 0, len(userIDs)) + ) + + switch o.fullUserCache { + case true: + for _, userID := range userIDs { + if _, ok := o.mapCache.Load(userID); ok { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + case false: + userOnlineMap, err := o.getUserOnlinePlatformBatch(ctx, userIDs) + if err != nil { + return nil, nil, err + } + + for key, value := range userOnlineMap { + if len(value) > 0 { + onlineUserIDs = append(onlineUserIDs, key) + } else { + offlineUserIDs = append(offlineUserIDs, key) + } + } + } + + log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return userIDs, offlineUserIDs, nil +} + //func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { // onlineUserIDs := make([]string, 0, len(userIDs)) // for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) +// online, err := o.GetUserOnline(ctx, userID) // if err != nil { // return nil, err // } @@ -111,6 +316,15 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e // return onlineUserIDs, nil //} -func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { - return o.local.SetHas(userID, platformIDs) +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { + switch o.fullUserCache { + case true: + o.mapCache.Store(userID, platformIDs) + case false: + o.lruCache.Set(userID, platformIDs) + } +} + +func (o *OnlineCache) setHasUserOnline(userID string, platformIDs []int32) bool { + return o.lruCache.SetHas(userID, platformIDs) } diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index eabe77b942..375cc993cb 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -169,6 +169,15 @@ func (u *UserRpcClient) Access(ctx context.Context, ownerUserID string) error { return authverify.CheckAccessV3(ctx, ownerUserID, u.imAdminUserID) } +// GetAllUserID retrieves all user IDs with pagination options. +func (u *UserRpcClient) GetAllUserID(ctx context.Context, pageNumber, showNumber int32) (*user.GetAllUserIDResp, error) { + resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) + if err != nil { + return nil, err + } + return resp, nil +} + // GetAllUserIDs retrieves all user IDs with pagination options. func (u *UserRpcClient) GetAllUserIDs(ctx context.Context, pageNumber, showNumber int32) ([]string, error) { resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) @@ -215,3 +224,7 @@ func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string } return resp[0].PlatformIDs, nil } + +func (u *UserRpcClient) GetAllOnlineUsers(ctx context.Context, cursor uint64) (*user.GetAllOnlineUsersResp, error) { + return u.Client.GetAllOnlineUsers(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) +} diff --git a/start-config.yml b/start-config.yml index a6d3e47afa..1231b5d0d4 100644 --- a/start-config.yml +++ b/start-config.yml @@ -3,8 +3,8 @@ serviceBinaries: openim-crontask: 1 openim-rpc-user: 1 openim-msggateway: 1 - openim-push: 4 - openim-msgtransfer: 4 + openim-push: 8 + openim-msgtransfer: 8 openim-rpc-conversation: 1 openim-rpc-auth: 1 openim-rpc-group: 1 From 80c71b77d6275baaaf82ab117406a412f6494309 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:12:02 +0800 Subject: [PATCH 91/91] feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) --- go.mod | 3 +-- go.sum | 22 ---------------------- internal/msggateway/client.go | 5 ++++- internal/msggateway/constant.go | 1 + internal/msggateway/message_handler.go | 25 +++++++++++++++++++++++-- pkg/rpcclient/msg.go | 17 ++++++++++++++--- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index cec96b5882..fa1d87922b 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require github.com/google/uuid v1.6.0 require ( github.com/IBM/sarama v1.43.0 github.com/fatih/color v1.14.1 + github.com/gin-contrib/gzip v1.0.1 github.com/go-redis/redis v6.15.9+incompatible github.com/go-redis/redismock/v9 v9.2.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -76,7 +77,6 @@ require ( github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect @@ -91,7 +91,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/gzip v1.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index be7b7b3ad5..1e9a371506 100644 --- a/go.sum +++ b/go.sum @@ -65,9 +65,6 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -75,9 +72,6 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -154,8 +148,6 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= -github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= -github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -269,8 +261,6 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -303,8 +293,6 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -427,8 +415,6 @@ github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -477,8 +463,6 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -488,8 +472,6 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -518,8 +500,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -613,8 +593,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 5a274f9e7a..bc06fa9507 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -22,6 +22,8 @@ import ( "sync/atomic" "time" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" @@ -30,7 +32,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/stringutil" - "google.golang.org/protobuf/proto" ) var ( @@ -222,6 +223,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.PullMessageBySeqList(ctx, binaryReq) case WSPullMsg: resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) + case WSGetConvMaxReadSeq: + resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 154be014e9..584cebe1e1 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -40,6 +40,7 @@ const ( WSSendMsg = 1003 WSSendSignalMsg = 1004 WSPullMsg = 1005 + WSGetConvMaxReadSeq = 1006 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 2f9620ce16..4b78c10048 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -19,6 +19,8 @@ import ( "sync" "github.com/go-playground/validator/v10" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/msg" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/jsonutil" - "google.golang.org/protobuf/proto" ) type Req struct { @@ -94,6 +95,7 @@ type MessageHandler interface { SendMessage(context context.Context, data *Req) ([]byte, error) SendSignalMessage(context context.Context, data *Req) ([]byte, error) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) + GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) GetSeqMessage(context context.Context, data *Req) ([]byte, error) UserLogout(context context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(context context.Context, data *Req) ([]byte, bool, error) @@ -176,7 +178,7 @@ func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]by func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) { req := sdkws.PullMessageBySeqsReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { - return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") } if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "PullMessageBySeqsReq") @@ -192,6 +194,25 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ return c, nil } +func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) { + req := msg.GetConversationsHasReadAndMaxSeqReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetConversationsHasReadAndMaxSeq") + } + resp, err := g.msgRpcClient.GetConversationsHasReadAndMaxSeq(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + return c, nil +} + func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, error) { req := msg.GetSeqMessageReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 5a06dac5db..9b26a7abda 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -17,6 +17,11 @@ package rpcclient import ( "context" "encoding/json" + "time" + + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -28,9 +33,6 @@ import ( "github.com/openimsdk/tools/utils/idutil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/timeutil" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - "time" ) func newContentTypeConf(conf *config.Notification) map[int32]config.NotificationConfig { @@ -221,6 +223,15 @@ func (m *MessageRpcClient) PullMessageBySeqList(ctx context.Context, req *sdkws. return resp, nil } +func (m *MessageRpcClient) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *msg.GetConversationsHasReadAndMaxSeqReq) (*msg.GetConversationsHasReadAndMaxSeqResp, error) { + resp, err := m.Client.GetConversationsHasReadAndMaxSeq(ctx, req) + if err != nil { + // Wrap the error to provide more context if the gRPC call fails. + return nil, err + } + return resp, nil +} + func (m *MessageRpcClient) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { return m.Client.GetSeqMessage(ctx, req) }