diff --git a/.env b/.env index 1e7b1e11a7..8863f2ec1e 100644 --- a/.env +++ b/.env @@ -1,13 +1,15 @@ - MONGO_IMAGE=mongo:6.0.2 REDIS_IMAGE=redis:7.0.0 ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 +PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 +ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 +GRAFANA_IMAGE=grafana/grafana:11.0.1 -OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.5.1 -OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.7 +OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.0 +OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.0 #FRONT_IMAGE: use aliyun images #OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.5.1 diff --git a/.github/.codecov.yml b/.github/.codecov.yml index fab584a316..c14ba471f5 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -1,17 +1,3 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - coverage: status: project: @@ -28,6 +14,7 @@ coverage: paths: - test/* # only include coverage in "test/" folder informational: true # Always pass check + # internal: # declare a new status context "internal" # paths: # - internal/* # only include coverage in "internal/" folder diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000000..2158804b76 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,65 @@ +name: Bug Report +title: "[BUG] " +labels: ["bug"] +description: "Create a detailed report to help us identify and resolve issues." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to fill out the bug report. Please provide as much information as possible to help us understand and replicate the bug." + + - type: input + id: openim-server-version + attributes: + label: OpenIM Server Version + description: "Please provide the version number of OpenIM Server you are using." + placeholder: "e.g., 3.8.0" + validations: + required: true + + - type: dropdown + id: operating-system + attributes: + label: Operating System and CPU Architecture + description: "Please select the operating system and describe the CPU architecture." + options: + - Linux (AMD) + - Linux (ARM) + - Windows (AMD) + - Windows (ARM) + - macOS (AMD) + - macOS (ARM) + validations: + required: true + + - type: dropdown + id: deployment-method + attributes: + label: Deployment Method + description: "Please specify how OpenIM Server was deployed." + options: + - Source Code Deployment + - Docker Deployment + validations: + required: true + + - type: textarea + id: bug-description-reproduction + attributes: + label: Bug Description and Steps to Reproduce + description: "Provide a detailed description of the bug and a step-by-step guide on how to reproduce it." + placeholder: "Describe the bug in detail here...\n\nSteps to reproduce the bug on the server:\n1. Start the server with specific configurations (mention any relevant config details).\n2. Make an API call to '...' endpoint with the following payload '...'.\n3. Observe the behavior and note any error messages or logs.\n4. Mention any additional setup relevant to the bug (e.g., database version, external service dependencies)." + validations: + required: true + + - type: markdown + attributes: + value: "If possible, please add screenshots to help explain your problem." + + - type: textarea + id: screenshots-link + attributes: + label: Screenshots Link + description: "If applicable, please provide any links to screenshots here." + placeholder: "Paste your screenshot URL here, e.g., http://imgur.com/example" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..deb8990831 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + # - name: "Bug Report" + # description: "Report a bug in the project" + # file: "bug-report.yml" + - name: πŸ“’ Connect on slack + url: https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg + about: Support OpenIM-related requests or issues, get in touch with developers and help on slack + - name: 🌐 OpenIM Blog + url: https://www.openim.io/ + about: Open the OpenIM community blog diff --git a/.github/ISSUE_TEMPLATE/deployment.yml b/.github/ISSUE_TEMPLATE/deployment.yml new file mode 100644 index 0000000000..8df6243228 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/deployment.yml @@ -0,0 +1,65 @@ +name: Deployment issue +title: "[Deployment] " +labels: ["deployment"] +description: "Create a detailed report to help us identify and resolve deployment issues." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to fill out the deployment issue report. Please provide as much information as possible to help us understand and resolve the issue." + + - type: input + id: openim-server-version + attributes: + label: OpenIM Server Version + description: "Please provide the version number of OpenIM Server you are using." + placeholder: "e.g., 3.8.0" + validations: + required: true + + - type: dropdown + id: operating-system + attributes: + label: Operating System and CPU Architecture + description: "Please select the operating system and describe the CPU architecture." + options: + - Linux (AMD) + - Linux (ARM) + - Windows (AMD) + - Windows (ARM) + - macOS (AMD) + - macOS (ARM) + validations: + required: true + + - type: dropdown + id: deployment-method + attributes: + label: Deployment Method + description: "Please specify how OpenIM Server was deployed." + options: + - Source Code Deployment + - Docker Deployment + validations: + required: true + + - type: textarea + id: issue-description-reproduction + attributes: + label: Issue Description and Steps to Reproduce + description: "Provide a detailed description of the issue and a step-by-step guide on how to reproduce it." + placeholder: "Describe the issue in detail here...\n\nSteps to reproduce the issue on the server:\n1. Start the server with specific configurations (mention any relevant config details).\n2. Make an API call to '...' endpoint with the following payload '...'.\n3. Observe the behavior and note any error messages or logs.\n4. Mention any additional setup relevant to the bug (e.g., database version, external service dependencies)." + validations: + required: true + + - type: markdown + attributes: + value: "If possible, please add screenshots to help explain your problem." + + - type: textarea + id: screenshots-link + attributes: + label: Screenshots Link + description: "If applicable, please provide any links to screenshots here." + placeholder: "Paste your screenshot URL here, e.g., http://imgur.com/example" \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..e6f751e4eb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,20 @@ +--- +name: Documentation Update +about: Propose updates to documentation, including README files and other docs. +title: "[DOC]: " # Prefix for the title to help identify documentation issues +labels: documentation # Labels to be automatically added +assignees: '' # Optionally, specify maintainers or teams to be auto-assigned + +--- + +## Documentation Updates +Describe the documentation that needs to be updated or corrected. Please specify the files and sections if possible. + +## Motivation +Explain why these updates are necessary. What is missing, misleading, or outdated? + +## Suggested Changes +Detail the changes that you propose. If you are suggesting large changes, include examples or mockups of what the updated documentation should look like. + +## Additional Information +Include any other information that might be relevant, such as links to discussions or related issues in the repository. diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000000..18a96a965c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,43 @@ +name: Feature Request +title: "[FEATURE REQUEST] " +labels: ["feature request","enhancement"] +description: "Propose a new feature or improvement that you believe will help enhance the project." +# assignees: [] + +body: + - type: markdown + attributes: + value: "Thank you for taking the time to propose a feature request. Please fill in as much detail as possible to help us understand why this feature is necessary and how it should work." + + - type: textarea + id: feature-reason + attributes: + label: Why this feature? + description: "Explain why this feature is needed. What problem does it solve? How does it benefit the project and its users?" + placeholder: "Describe the need for this feature..." + validations: + required: true + + - type: textarea + id: solution-proposal + attributes: + label: Suggested Solution + description: "Describe your proposed solution for this feature. How do you envision it working?" + placeholder: "Detail your solution here..." + validations: + required: true + + - type: markdown + attributes: + value: "Please provide any other relevant information or screenshots that could help illustrate your idea." + + - type: textarea + id: additional-info + attributes: + label: Additional Information + description: "Include any additional information, links, or screenshots that might be relevant to your feature request." + placeholder: "Add more context or links to relevant resources..." + + - type: markdown + attributes: + value: "Thank you for contributing to the project! We appreciate your input and will review your suggestion as soon as possible." diff --git a/.github/ISSUE_TEMPLATE/other.yml b/.github/ISSUE_TEMPLATE/other.yml new file mode 100644 index 0000000000..025440229a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other.yml @@ -0,0 +1,30 @@ +name: 🐧 Other +description: Use this for any other issues. Please do NOT create blank issues +title: "[Other]: " +labels: ["other"] +# assignees: [] + + +body: + - type: markdown + attributes: + value: "# Other issue" + - type: textarea + id: issuedescription + attributes: + label: What would you like to share? + description: Provide a clear and concise explanation of your issue. + validations: + required: true + - type: textarea + id: extrainfo + attributes: + label: Additional information + description: Is there anything else we should know about this issue? + validations: + required: false + - type: markdown + attributes: + value: | + You can also join our Discord community [here](https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg) + Feel free to check out other cool repositories of the openim Community [here](https://github.com/openimsdk) diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md new file mode 100644 index 0000000000..760d89e534 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -0,0 +1,26 @@ +--- +name: RFC - Feature Proposal +about: Submit a proposal for a significant feature to invite community discussion. +title: "[RFC]: " # Prefix for the title to help identify RFC proposals +labels: rfc, proposal # Labels to be automatically added +assignees: '' # Optionally, specify maintainers or teams to be auto-assigned + +--- + +## Proposal Overview +Briefly describe the content and objectives of your proposal. + +## Motivation +Why is this new feature necessary? What is the background of this problem? + +## Detailed Design +Describe the technical details of the proposal, including implementation steps, code snippets, or architecture diagrams. + +## Alternatives Considered +Have other alternatives been considered? Why is this approach preferred over others? + +## Impact +How will this proposal affect existing practices and community users? + +## Additional Information +Include any other relevant information such as related discussions, prior related work, etc. diff --git a/.github/code-language-detector.yml b/.github/code-language-detector.yml deleted file mode 100644 index 194c2474ad..0000000000 --- a/.github/code-language-detector.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Β© 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/marketplace/actions/code-language-detector -directory: ./ -file_types: - - .go - - .yaml - - .yml -languages: - - Chinese \ No newline at end of file diff --git a/.github/labels.yml b/.github/labels.yml deleted file mode 100644 index b85a824b42..0000000000 --- a/.github/labels.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Refer to Kubernetes for size/* Settings -# https://github.com/Kubernetes/Kubernetes -XS: - name: size/XS - lines: 0 - color: 3CBF00 -S: - name: size/S - lines: 10 - color: 5D9801 -M: - name: size/M - lines: 30 - color: 7F7203 -L: - name: size/L - lines: 100 - color: A14C05 -XL: - name: size/XL - lines: 500 - color: C32607 -XXL: - name: size/XXL - lines: 1000 - color: E50009 - comment: | - # Whoa! Easy there, Partner! - This PR is too big. Please break it up into smaller PRs. \ No newline at end of file diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml deleted file mode 100644 index 55ee241d7b..0000000000 --- a/.github/release-drafter.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name-template: 'v$RESOLVED_VERSION 🌈' -tag-template: 'v$RESOLVED_VERSION' -categories: - - title: 'πŸš€ Features' - labels: - - 'feature' - - 'enhancement' - - title: 'πŸ› Bug Fixes' - labels: - - 'kind/fix' - - 'kind/feature' - - 'enhancement' - - 'kind/documentation' - - 'good first issue' - - title: '🧰 Maintenance' - label: 'chore' -change-template: '- $TITLE @$AUTHOR (#$NUMBER)' -change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. -version-resolver: - major: - labels: - - 'major' - minor: - labels: - - 'minor' - patch: - labels: - - 'patch' - default: patch -template: | - ## Changes $PREVIOUS_TAG - - $CHANGES - - ## Contributors to this $REPOSITORY release - - $CONTRIBUTORS diff --git a/.github/standardizer.yml b/.github/standardizer.yml deleted file mode 100644 index fceb69df10..0000000000 --- a/.github/standardizer.yml +++ /dev/null @@ -1,50 +0,0 @@ -# https://github.com/marketplace/actions/conformity-checker-for-project -baseConfig: - searchDirectory: "./" - ignoreCase: false - -directoryNaming: - allowHyphens: true - allowUnderscores: false - mustBeLowercase: true - -fileNaming: - allowHyphens: true - allowUnderscores: true - mustBeLowercase: true - -ignoreFormats: - - "\\.log$" - - "\\.env$" - - "README\\.md$" - - "_test\\.go$" - - "\\.md$" - - _test\\.txt$ - - LICENSE - - Dockerfile - - CODEOWNERS - - Makefile - -ignoreDirectories: - - "vendor" - - ".git" - - "deployments" - - "node_modules" - - "logs" - - "CHANGELOG" - - "components" - - "_output" - - "tools/openim-web" - - "CHANGELOG" - - "examples/Test_directory" - - test/testdata - -fileTypeSpecificNaming: - ".yaml": - allowHyphens: true - allowUnderscores: false - mustBeLowercase: true - ".go": - allowHyphens: false - allowUnderscores: true - mustBeLowercase: true \ No newline at end of file diff --git a/.github/sync-release.yml b/.github/sync-release.yml index 3800b4c24b..18a80fda2c 100644 --- a/.github/sync-release.yml +++ b/.github/sync-release.yml @@ -1,18 +1,4 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OpenIMSDK/openim-docker: +openimsdk/openim-docker: - source: ./config dest: ./openim-server/release/config replace: true diff --git a/.github/sync.yml b/.github/sync.yml deleted file mode 100644 index ee667d415f..0000000000 --- a/.github/sync.yml +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -# Synchronization for the.github repository -OpenIMSDK/.github: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - -OpenIMSDK/community: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/ - dest: .github/workflows/ - -OpenIMSDK/openim-sdk-core: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/OpenIM-Docs: - - source: .github/workflows/ - dest: .github/workflows/ - exclude: | - e2e-test.yml - sync.yml - - source: scripts/githooks/ - dest: scripts/githooks/ - replace: true - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/OpenKF: - - source: LICENSE - dest: LICENSE - - source: scripts/LICENSE/ - dest: scripts/LICENSE/ - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - -OpenIMSDK/openim-docker: - - source: ./config - dest: ./openim-server/main/config - replace: true - - source: ./docs - dest: ./openim-server/main/docs - replace: true - - source: ./scripts - dest: ./openim-server/main/scripts - replace: true - - source: ./scripts - dest: ./scripts - replace: true - - source: ./Makefile - dest: ./Makefile - replace: true - -group: - # first group：common to all warehouses - # TODO: add the required warehouse here - - repos: | - OpenIMSDK/OpenKF@main - OpenIMSDK/openim-miniprogram-demo@main - OpenIMSDK/docs - OpenIMSDK/chat - OpenIMSDK/community - OpenIMSDK/openim-charts - OpenIMSDK/openim-sdk-cpp@main - files: - - source: LICENSE - dest: LICENSE - replace: false - - source: .github/workflows/issue-robot.yml - dest: .github/workflows/issue-robot.yml - replace: false - - source: .github/workflows/stale.yml - dest: .github/workflows/stale.yml - replace: false - - source: .github/workflows/project-progress.yml - dest: .github/workflows/project-progress.yml - replace: false - - source: .github/workflows/help-comment-issue.yml - dest: .github/workflows/help-comment-issue.yml - replace: false - - source: .github/.codecov.yml - dest: .github/.codecov.yml - replace: false - - source: .github/workflows/cla.yml - dest: .github/workflows/cla.yml - replace: false - - source: .github/workflows/auto-assign-issue.yml - dest: .github/workflows/auto-assign-issue.yml - replace: false - - source: .github/workflows/release.yml - dest: .github/workflows/release.yml - replace: false - - source: ./scripts/githooks/ - dest: ./scripts/githooks/ - replace: true diff --git a/.github/weekly-digest.yml b/.github/weekly-digest.yml deleted file mode 100644 index fb3614ad81..0000000000 --- a/.github/weekly-digest.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/apps/weekly-digest/installations/new -publishDay: sun -canPublishIssues: true -canPublishPullRequests: true -canPublishContributors: true -canPublishStargazers: true -canPublishCommits: true \ No newline at end of file diff --git a/.github/workflows/auto-assign-issue.yml b/.github/workflows/auto-assign-issue.yml index d92fc968cb..320174d8c9 100644 --- a/.github/workflows/auto-assign-issue.yml +++ b/.github/workflows/auto-assign-issue.yml @@ -1,17 +1,3 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - name: Assign issue to comment author on: issue_comment: @@ -20,8 +6,7 @@ jobs: assign-issue: if: | contains(github.event.comment.body, '/assign') || contains(github.event.comment.body, '/accept') && - !contains(github.event.comment.user.login, 'openimbot') && - !contains(github.event.comment.user.login, 'kubbot') + !contains(github.event.comment.user.login, 'openim-robot') runs-on: ubuntu-latest permissions: issues: write @@ -33,11 +18,12 @@ jobs: run: | export LETASE_MILESTONES=$(curl 'https://api.github.com/repos/$OWNER/$PEPO/milestones' | jq -r 'last(.[]).title') gh issue edit ${{ github.event.issue.number }} --add-assignee "${{ github.event.comment.user.login }}" - gh issue edit ${{ github.event.issue.number }} --add-label "triage/accepted" - gh issue edit ${{ github.event.issue.number }} --milestone "$LETASE_MILESTONES" + gh issue edit ${{ github.event.issue.number }} --add-label "accepted" gh issue comment $ISSUE --body "@${{ github.event.comment.user.login }} Glad to see you accepted this issue🀲, this issue has been assigned to you. I set the milestones for this issue to [$LETASE_MILESTONES](https://github.com/$OWNER/$PEPO/milestones), We are looking forward to your PR!" + + # gh issue edit ${{ github.event.issue.number }} --milestone "$LETASE_MILESTONES" env: - GH_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.BOT_TOKEN }} ISSUE: ${{ github.event.issue.html_url }} OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} \ No newline at end of file + REPO: ${{ github.event.repository.name }} diff --git a/.github/workflows/auto-gh-pr.yml b/.github/workflows/auto-gh-pr.yml deleted file mode 100644 index 45454275ed..0000000000 --- a/.github/workflows/auto-gh-pr.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Auto PR to release - -on: - pull_request: - # types: - # - closed - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - -jobs: - sync-issue-to-pr: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Sync Issue to PR - if: github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main' - run: | - PR_BODY="${{ github.event.pull_request.body }}" - - ISSUE_NUMBER=$(echo "$PR_BODY" | grep -oP 'Fixes #\K\d+') - if [[ -z "$ISSUE_NUMBER" ]]; then - echo "No Issue number found." - exit 1 - fi - - echo "Issue number found: $ISSUE_NUMBER" - - # Using GitHub CLI to get issue details - gh issue view "$ISSUE_NUMBER" --repo "${{ github.repository }}" --json labels,assignees,milestone,title > issue_data.json - - # Check if jq is installed - if ! command -v jq &> /dev/null; then - echo "Installing jq..." - sudo apt-get install -y jq - fi - - # Parse data with jq - LABELS=$(jq -r '.labels | map(.name) | join(",")' issue_data.json) - ASSIGNEES=$(jq -r '.assignees | map(.login) | join(",")' issue_data.json) - MILESTONE=$(jq -r '.milestone.title' issue_data.json) - - # Check if any of the fields are empty and set them to None - LABELS=${LABELS:-None} - ASSIGNEES=${ASSIGNEES:-None} - MILESTONE=${MILESTONE:-None} - - # Edit the PR with issue details, handling empty fields - gh pr edit "${{ github.event.pull_request.number }}" --repo "${{ github.repository }}" \ - ${LABELS:+--add-label "$LABELS"} \ - ${ASSIGNEES:+--add-assignee "$ASSIGNEES"} \ - ${MILESTONE:+--milestone "$MILESTONE"} - continue-on-error: true - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/auto-invite.yml b/.github/workflows/auto-invite-comment.yml similarity index 74% rename from .github/workflows/auto-invite.yml rename to .github/workflows/auto-invite-comment.yml index 350de30abc..76fbcdfd33 100644 --- a/.github/workflows/auto-invite.yml +++ b/.github/workflows/auto-invite-comment.yml @@ -1,32 +1,18 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Invite users to join our group +name: Invite users to join OpenIM Community. on: issue_comment: types: - created jobs: issue_comment: - name: Invite users to join our group + name: Invite users to join OpenIM Community if: ${{ github.event.comment.body == '/invite' || github.event.comment.body == '/close' || github.event.comment.body == '/comment' }} runs-on: ubuntu-latest permissions: issues: write steps: - - name: Invite user to join our group + - name: Invite user to join OpenIM Community uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.BOT_GITHUB_TOKEN }} @@ -43,11 +29,11 @@ jobs: + Read our [blog](https://doc.rentsoft.cn/). Our blog is a great place to stay up-to-date with Open-IM-Server projects and trends. On the blog, we share our latest developments, tech trends, and other interesting information. + Add [Wechat](https://github.com/OpenIMSDK/OpenIM-Docs/blob/main/docs/images/WechatIMG20.jpeg) and indicate that you are a user or developer of Open-IM-Server. We will process your request as soon as possible. - - name: Close Issue - uses: peter-evans/close-issue@v3 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - issue-number: ${{ github.event.issue.number }} - comment: πŸ€– Auto-closing issue, if you still need help please reopen the issue or ask for help in the community above - labels: | - triage/accepted \ No newline at end of file + # - name: Close Issue + # uses: peter-evans/close-issue@v3 + # with: + # token: ${{ secrets.BOT_GITHUB_TOKEN }} + # issue-number: ${{ github.event.issue.number }} + # comment: πŸ€– Auto-closing issue, if you still need help please reopen the issue or ask for help in the community above + # labels: | + # accepted \ No newline at end of file diff --git a/.github/workflows/auto-tag.yml b/.github/workflows/auto-tag.yml deleted file mode 100644 index 3decba7abc..0000000000 --- a/.github/workflows/auto-tag.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Create Tag - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - -jobs: - create_tag: - runs-on: ubuntu-latest - if: startsWith(github.event.comment.body, '/create tag') - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Validate version number and get comment - id: validate - run: | - COMMENT="${{ github.event.comment.body }}" - VERSION=$(echo $COMMENT | cut -d ' ' -f 3) - TAG_COMMENT=$(echo $COMMENT | cut -d '"' -f 2) - if [[ $VERSION =~ ^v([0-9]+\.){2}[0-9]+$ ]]; then - echo "version=$VERSION" >> $GITHUB_STATE - echo "tag_comment=$TAG_COMMENT" >> $GITHUB_STATE - else - echo "Invalid version number." - exit 1 - fi - - - name: Create a new tag - env: - GH_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - run: | - source $GITHUB_STATE - git tag -a $VERSION -m "$tag_comment" - git push origin $VERSION - echo "tag_created=$VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/bot-auto-cherry-pick.yml b/.github/workflows/bot-auto-cherry-pick.yml deleted file mode 100644 index cdd7241e2c..0000000000 --- a/.github/workflows/bot-auto-cherry-pick.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Rebot for Cherry Pick when PR is merged -on: - pull_request_target: - types: - - closed - -jobs: - comment: - runs-on: ubuntu-latest - steps: - - name: Comment cherry-pick command - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - script: | - const pr = context.payload.pull_request; - if (!pr.merged) { - console.log("PR is not merged. Skipping..."); - return; - } - if (!pr.milestone || !pr.milestone.title) { - console.log("Milestone is not set. Skipping..."); - return; - } - const milestone = pr.milestone.title; - const ref = `heads/release-${milestone}`; - let branchExists; - try { - await github.rest.git.getRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: ref - }); - branchExists = true; - } catch (error) { - if (error.status === 404) { - console.log(`Branch ${ref} does not exist. Skipping...`); - branchExists = false; - } else { - throw error; // Rethrow if it's another error - } - } - if (!branchExists) { - return; - } - const cherryPickCmd = `/cherry-pick release-${milestone}`; - console.log(`Adding comment: ${cherryPickCmd}`); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - body: cherryPickCmd - }); \ No newline at end of file diff --git a/.github/workflows/bot-cherry-pick.yml b/.github/workflows/bot-cherry-pick.yml deleted file mode 100644 index 71597189c6..0000000000 --- a/.github/workflows/bot-cherry-pick.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Robot for Cherry Pick On Comment - -on: - issue_comment: - types: [created] - -jobs: - cherry-pick: - name: Cherry Pick - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/cherry-pick') - runs-on: ubuntu-latest - - steps: - - name: Checkout the latest code - uses: actions/checkout@v4 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - fetch-depth: 0 # To ensure all history is available for cherry-picking - - - name: Automatic Cherry Pick - uses: vendoo/gha-cherry-pick@v1 - with: - # Assuming the cherry-pick commit SHA is passed in the comment like '/cherry-pick sha' - commit-sha: ${{ github.event.comment.body }} - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - - name: Create a new branch for PR - run: | - PR_BRANCH="cherry-pick-${GITHUB_SHA}-to-${{ github.base_ref }}" - git checkout -b $PR_BRANCH - git push origin $PR_BRANCH - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - - name: Create Pull Request - uses: actions/github-script@v5 - with: - script: | - const prTitle = "Cherry-pick to ${{ github.base_ref }}" - const prBody = "Automated cherry-pick of ${{ github.event.comment.body }}\n\n/cc @kubbot" - const base = "${{ github.base_ref }}" - const head = "cherry-pick-${{ github.sha }}-to-${{ github.base_ref }}" - const createPr = await github.rest.pulls.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: prTitle, - body: prBody, - head: head, - base: base, - maintainer_can_modify: true, // Allows maintainers to edit the PR - }) - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml deleted file mode 100644 index b4733116e0..0000000000 --- a/.github/workflows/build-docker-image.yml +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Publish Docker image - -on: - push: - branches: - - main - - release-* - tags: - - v* - workflow_dispatch: - -env: - # Common versions - GO_VERSION: "1.20" - -jobs: - build-dockerhub: - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - -# docker.io/openim/openim-server:latest - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5.5.1 - with: - images: openim/openim-server - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - # linux/ppc64le,linux/s390x - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-aliyun: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 -# registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server:latest - - name: Extract metadata (tags, labels) for Docker - id: meta2 - uses: docker/metadata-action@v5.5.1 - with: - images: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Log in to AliYun Docker Hub - uses: docker/login-action@v3 - with: - registry: registry.cn-hangzhou.aliyuncs.com - username: ${{ secrets.ALIREGISTRY_USERNAME }} - password: ${{ secrets.ALIREGISTRY_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - # linux/ppc64le,linux/s390x - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta2.outputs.tags }} - labels: ${{ steps.meta2.outputs.labels }} - - build-ghcr: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 -# ghcr.io/openimsdk/openim-server:latest - - name: Extract metadata (tags, labels) for Docker - id: meta3 - uses: docker/metadata-action@v5.5.1 - with: - images: ghcr.io/openimsdk/openim-server - # generate Docker tags based on the following events/attributes - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - # linux/ppc64le,linux/s390x - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta3.outputs.tags }} - labels: ${{ steps.meta3.outputs.labels }} diff --git a/.github/workflows/check-coverage.bak b/.github/workflows/check-coverage.bak deleted file mode 100644 index 09d43d7cdf..0000000000 --- a/.github/workflows/check-coverage.bak +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Check Coverage - -on: - workflow_dispatch: - push: - branches: [ "main" ] - paths-ignore: - - "docs/**" - - "**/*.md" - - "**/*.yaml" - - "CONTRIBUTORS" - - "CHANGELOG/**" - pull_request: - branches: [ "*" ] - paths-ignore: - - "docs/**" - - "**/*.md" - - "**/*.yaml" - - "CONTRIBUTORS" - - "CHANGELOG/**" -env: - # Common versions - GO_VERSION: "1.20" - -jobs: - coverage: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Golang with cache - uses: magnetikonline/action-golang-cache@v4 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - run: sudo apt update && sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev - - - name: Run Cover - run: make cover - continue-on-error: true - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v4 diff --git a/.github/workflows/cla-assistant.yml b/.github/workflows/cla-assistant.yml new file mode 100644 index 0000000000..7d44b05eb4 --- /dev/null +++ b/.github/workflows/cla-assistant.yml @@ -0,0 +1,40 @@ +name: CLA Assistant +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened,closed,synchronize] + +# explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings +permissions: + actions: write + contents: write # this can be 'read' if the signatures are in remote repository + pull-requests: write + statuses: write + +jobs: + CLA-Assistant: + runs-on: ubuntu-latest + steps: + - name: "CLA Assistant" + if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' + uses: contributor-assistant/github-action@v2.4.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.BOT_TOKEN }} + with: + path-to-signatures: 'signatures/cla.json' + path-to-document: 'https://github.com/OpenIM-Robot/cla/blob/main/README.md' # e.g. a CLA or a DCO document + branch: 'main' + allowlist: 'bot*,*bot,OpenIM-Robot' + + # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken + remote-organization-name: OpenIM-Robot + remote-repository-name: cla + create-file-commit-message: 'Creating file for storing CLA Signatures' + # signed-commit-message: '$contributorName has signed the CLA in $owner/$repo#$pullRequestNo' + custom-notsigned-prcomment: 'πŸ’• Thank you for your contribution and please kindly read and sign our CLA. [CLA Docs](https://github.com/OpenIM-Robot/cla/blob/main/README.md)' + custom-pr-sign-comment: 'I have read the CLA Document and I hereby sign the CLA' + custom-allsigned-prcomment: 'πŸ€– All Contributors have signed the [CLA](https://github.com/OpenIM-Robot/cla/blob/main/README.md).
The signed information is recorded [**here**](https://github.com/OpenIM-Robot/cla/blob/main/signatures/cla.json)' + #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) + #use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml deleted file mode 100644 index ae27c8a0a5..0000000000 --- a/.github/workflows/cla.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM CLA Assistant -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened,closed,synchronize] - -# explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings -permissions: - actions: write - contents: write - pull-requests: write - statuses: write - -env: - # Define Open-IM-Server variables here - OPEN_IM_SERVER_REMOTE_ORGANIZATION: openim-sigs - REMOTE_REPOSITORY: cla - OPEN_IM_SERVER_CLA_DOCUMENT: https://github.com/openim-sigs/cla/blob/main/README.md - OPEN_IM_SERVER_SIGNATURES_PATH: signatures/${{ github.event.repository.name }}/cla.json - - OPEN_IM_SERVER_ALLOWLIST: kubbot,openimbot,bot*,dependabot,sweep-ai,*bot,bot-*,bot/*,bot-/*,bot,*[bot] - -jobs: - CLAAssistant: - runs-on: ubuntu-latest - steps: - - name: "CLA Assistant" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: contributor-assistant/github-action@v2.3.1 - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} - with: - path-to-signatures: ${{ env.OPEN_IM_SERVER_SIGNATURES_PATH }} - path-to-document: ${{ env.OPEN_IM_SERVER_CLA_DOCUMENT }} - branch: 'main' - allowlist: ${{ env.OPEN_IM_SERVER_ALLOWLIST }} - - remote-organization-name: ${{ env.OPEN_IM_SERVER_REMOTE_ORGANIZATION }} - remote-repository-name: ${{ env.REMOTE_REPOSITORY }} - - create-file-commit-message: 'πŸ“š Docs: Creating file for storing ${{ github.event.repository.name }} CLA Signatures' - custom-notsigned-prcomment: 'πŸ’• Thank you for your contribution and please kindly read and sign our [🎯https://github.com/openim-sigs/cla/blob/main/README.md](https://github.com/openim-sigs/cla/blob/main/README.md).
If you wish to sign the CRA, **Please copy and comment on the following sentence:**' - custom-pr-sign-comment: 'I have read the CLA Document and I hereby sign the CLA' - custom-allsigned-prcomment: 'πŸ€– All Contributors have signed the [${{ github.event.repository.name }} CLA](https://github.com/openim-sigs/cla/blob/main/README.md).
The signed information is recorded [πŸ€–here](https://github.com/openim-sigs/cla/tree/main/signatures/${{ github.event.repository.name }}/cla.json)' - # lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) - # use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/code-language-detector.yml b/.github/workflows/code-language-detector.yml deleted file mode 100644 index 80ec947338..0000000000 --- a/.github/workflows/code-language-detector.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Β© 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Language Check Workflow Test - -on: [pull_request] - -jobs: - comment-language-detector: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Code Language Detector - uses: kubecub/comment-lang-detector@v1.0.0 \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 29f9382ccc..fd871e2b58 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,76 +1,67 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. -name: "OpenIM Code Scanning - Action" +name: "CodeQL" on: push: - branches: [main] + branches: [ main ] pull_request: - branches: [main] + # The branches below must be a subset of the branches above + branches: [ main ] schedule: - # β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ minute (0 - 59) - # β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ hour (0 - 23) - # β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ day of the month (1 - 31) - # β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ month (1 - 12 or JAN-DEC) - # β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ day of the week (0 - 6 or SUN-SAT) - # β”‚ β”‚ β”‚ β”‚ β”‚ - # β”‚ β”‚ β”‚ β”‚ β”‚ - # β”‚ β”‚ β”‚ β”‚ β”‚ - # * * * * * - - cron: '30 1 * * 0' + - cron: '18 19 * * 6' jobs: - CodeQL-Build: - # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + analyze: + name: Analyze runs-on: ubuntu-latest - permissions: - # required for all workflows - security-events: write - - # only required for workflows in private repositories - actions: write - contents: write + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - # Override language selection by uncommenting this and choosing your languages - with: - languages: go + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below). - - name: Autobuild - uses: github/codeql-action/autobuild@v3 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 - # ℹ️ Command-line programs to run using the OS shell. - # πŸ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + # ℹ️ Command-line programs to run using the OS shell. + # πŸ“š https://git.io/JvXDl - # ✏️ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - # - run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 \ No newline at end of file + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 \ No newline at end of file diff --git a/.github/workflows/comment-check.yml b/.github/workflows/comment-check.yml new file mode 100644 index 0000000000..e994b52596 --- /dev/null +++ b/.github/workflows/comment-check.yml @@ -0,0 +1,51 @@ +name: Non-English Comments Check + +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + non-english-comments-check: + runs-on: ubuntu-latest + + env: + # need ignore Dirs + EXCLUDE_DIRS: ".git docs tests scripts assets node_modules build" + # need ignore Files + EXCLUDE_FILES: "*.md *.txt *.html *.css *.min.js *.mdx" + + steps: + - uses: actions/checkout@v4 + + - name: Search for Non-English comments + run: | + set -e + # Define the regex pattern to match Chinese characters + pattern='[\p{Han}]' + + # Process the directories to be excluded + exclude_dirs="" + for dir in $EXCLUDE_DIRS; do + exclude_dirs="$exclude_dirs --exclude-dir=$dir" + done + + # Process the file types to be excluded + exclude_files="" + for file in $EXCLUDE_FILES; do + exclude_files="$exclude_files --exclude=$file" + done + + # Use grep to find all comments containing Non-English characters and save to file + grep -Pnr "$pattern" . $exclude_dirs $exclude_files > non_english_comments.txt || true + + - name: Output non-English comments are found + run: | + if [ -s non_english_comments.txt ]; then + echo "Non-English comments found in the following locations:" + cat non_english_comments.txt + exit 1 # terminate the workflow + else + echo "No Non_English comments found." + fi diff --git a/.github/workflows/create-branch-on-tag.bak b/.github/workflows/create-branch-on-tag.bak deleted file mode 100644 index fbacd261b1..0000000000 --- a/.github/workflows/create-branch-on-tag.bak +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Create Branch on Tag - -on: - push: - tags: - - 'v*.*.0' - -permissions: - contents: write - actions: write - -jobs: - create-branch: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Git - run: | - git config --global user.name 'kubbot' - git config --global user.email '3293172751yxy@gmail.com' - - - name: Install git-chglog - run: make install.git-chglog - - - name: Create Branch and Push - env: - TAG_NAME: ${{ github.ref_name }} - run: | - IFS='.' read -ra VERSION_PARTS <<< "$TAG_NAME" - if [[ "${VERSION_PARTS[2]}" = "0" ]]; then - BRANCH_NAME="release-v${VERSION_PARTS[0]}.${VERSION_PARTS[1]}" - echo "Creating branch $BRANCH_NAME" - git checkout -b "$BRANCH_NAME" - git push origin "$BRANCH_NAME" - else - echo "Not a release tag. Skipping branch creation." - fi - continue-on-error: true - - - name: Create and Commit CHANGELOG - if: endsWith(github.ref_name, '.0') - run: | - git fetch --all - TAG_NAME=${GITHUB_REF#refs/tags/} - IFS='.' read -ra VERSION_PARTS <<< "$TAG_NAME" - git checkout main - cd CHANGELOG - git-chglog --tag-filter-pattern "v${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.*" -o "CHANGELOG-${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.md" - git add "CHANGELOG-${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.md" - git commit -m "Update CHANGELOG for $TAG_NAME" || echo "No changes to commit." - continue-on-error: true - - - name: Push CHANGELOG to Main - if: steps.create-and-commit-changelog.outputs.changes == 'true' - uses: ad-m/github-push-action@v0.8.0 - with: - github_token: ${{ secrets.BOT_GITHUB_TOKEN }} - branch: main - continue-on-error: true diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml deleted file mode 100644 index aff7e3d9e0..0000000000 --- a/.github/workflows/depsreview.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Β© 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -name: OpenIM Dependency Review -on: [pull_request] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v4 - - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 \ No newline at end of file diff --git a/.github/workflows/docker-buildx.bak b/.github/workflows/docker-buildx.bak deleted file mode 100644 index 7e7b8229ca..0000000000 --- a/.github/workflows/docker-buildx.bak +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Docker Buildx Images CI - -on: - schedule: - - cron: '30 1 * * *' - push: - branches: - - release-* - tags: - - v* - workflow_dispatch: - -jobs: - build-ghcr: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - install: true - - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Log in to AliYun Docker Hub - uses: docker/login-action@v3 - with: - registry: registry.cn-hangzhou.aliyuncs.com - username: ${{ secrets.ALIREGISTRY_USERNAME }} - password: ${{ secrets.ALIREGISTRY_TOKEN }} - -################################################ -# build/ -# └── docker -# β”œβ”€β”€ openim-api -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-cmdutils -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-crontask -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-msggateway -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-msgtransfer -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-push -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-auth -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-conversation -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-friend -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-group -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-msg -# β”‚ └── Dockerfile -# β”œβ”€β”€ openim-rpc-third -# β”‚ └── Dockerfile -# └── openim-rpc-user -# └── Dockerfile -############################################# - - - name: Extract metadata (tags, labels) for Docker openim-api - id: meta1 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-api - openim/openim-api - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-api - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-api - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-api/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta1.outputs.tags }} - labels: ${{ steps.meta1.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-cmdutils - id: meta2 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-cmdutils - openim/openim-cmdutils - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-cmdutils - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-cmdutils - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-cmdutils/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta2.outputs.tags }} - labels: ${{ steps.meta2.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-crontask - id: meta3 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-crontask - openim/openim-crontask - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-crontask - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-crontask - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-crontask/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta3.outputs.tags }} - labels: ${{ steps.meta3.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-msggateway - id: meta4 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-msggateway - openim/openim-msggateway - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-msggateway - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-msggateway - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-msggateway/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta4.outputs.tags }} - labels: ${{ steps.meta4.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-msgtransfer - id: meta5 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-msgtransfer - openim/openim-msgtransfer - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-msgtransfer - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-msgtransfer - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-msgtransfer/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta5.outputs.tags }} - labels: ${{ steps.meta5.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-push - id: meta6 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-push - openim/openim-push - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-push - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-push - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-push/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta6.outputs.tags }} - labels: ${{ steps.meta6.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-auth - id: meta7 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-auth - openim/openim-rpc-auth - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-auth - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-auth - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-auth/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta7.outputs.tags }} - labels: ${{ steps.meta7.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-conversation - id: meta8 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-conversation - openim/openim-rpc-conversation - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-conversation - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-conversation - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-conversation/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta8.outputs.tags }} - labels: ${{ steps.meta8.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-friend - id: meta9 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-friend - openim/openim-rpc-friend - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-friend - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-friend - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-friend/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta9.outputs.tags }} - labels: ${{ steps.meta9.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-group - id: meta10 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-group - openim/openim-rpc-group - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-group - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-group - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-group/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta10.outputs.tags }} - labels: ${{ steps.meta10.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-msg - id: meta11 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-msg - openim/openim-rpc-msg - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-msg - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-msg - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-msg/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta11.outputs.tags }} - labels: ${{ steps.meta11.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-third - id: meta12 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-third - openim/openim-rpc-third - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-third - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-third - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-third/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta12.outputs.tags }} - labels: ${{ steps.meta12.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Extract metadata (tags, labels) for Docker openim-rpc-user - id: meta13 - uses: docker/metadata-action@v5.5.1 - with: - images: | - ghcr.io/openimsdk/openim-rpc-user - openim/openim-rpc-user - registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-rpc-user - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - - - name: Build and push Docker image for openim-rpc-user - uses: docker/build-push-action@v5 - with: - context: . - file: ./build/images/openim-rpc-user/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta13.outputs.tags }} - labels: ${{ steps.meta13.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/e2e-test.bak b/.github/workflows/e2e-test.bak deleted file mode 100644 index 6231697c25..0000000000 --- a/.github/workflows/e2e-test.bak +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM E2E And API Test - -on: - workflow_dispatch: - pull_request: - push: - schedule: - # run e2e test every 4 hours - - cron: 0 */4 * * * - -env: - CALLBACK_ENABLE: true - -jobs: - build: - name: Test - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - - name: Set up Go 1.21 - uses: actions/setup-go@v5 - with: - go-version: 1.21 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Create e2e test - run: | - echo "...test e2e" - - execute-linux-systemd-scripts: - name: Execute OpenIM script on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - environment: - name: openim - strategy: - matrix: - go_version: ["1.20"] - os: ["ubuntu-latest"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go ${{ matrix.go_version }} - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go_version }} - id: go - - - name: Install Task - uses: arduino/setup-task@v1 - with: - version: '3.x' # If available, use the latest major version that's compatible - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Operations - run: | - sudo docker compose up -d - sudo bash bootstrap.sh - sudo mage - sudo sleep 20 - - - name: Module Operations - run: | - echo "===========> Verifying go-gitlint is installed" - if [ ! -f ./_output/tools/go-gitlint ]; then - export GOBIN=$(pwd)/_output/tools - echo "===========> Installing The default installation path is /home/ubuntu/DF/open-im-server/_output/tools/go-gitlint" - sudo go install github.com/marmotedu/go-gitlint/cmd/go-gitlint@latest - echo "===========> go-gitlint is installed in /home/ubuntu/DF/open-im-server/_output/tools/go-gitlint" - fi - - - name: Build, Start(make build && make start) - run: | - sudo ./scripts/install/install.sh -i - - - name: Exec OpenIM System Status Chack - run: | - sudo ./scripts/install/install.sh -s - -# - name: Exec OpenIM API test (make test-api) - - name: Exec OpenIM test (make test) - run: | - mkdir -p ./tmp - touch ./tmp/test.md - echo "# OpenIM Test" >> ./tmp/test.md - echo "## OpenIM API Test" >> ./tmp/test.md - echo "
Command Output for OpenIM API Test" >> ./tmp/test.md - echo "
" >> ./tmp/test.md
-        echo "===========> Run api test"
-        ./scripts/install/test.sh
-        echo "===========> Run api test" >> ./tmp/test.md
-        ./scripts/install/test.sh >> ./tmp/test.md
-        echo "
" >> ./tmp/test.md - echo "
" >> ./tmp/test.md - - echo "===========> Run api test" - ./scripts/install/test.sh - - # - name: Exec OpenIM E2E Test (make test-e2e) - # run: | - # echo "" >> ./tmp/test.md - # echo "## OpenIM E2E Test" >> ./tmp/test.md - # echo "
Command Output for OpenIM E2E Test" >> ./tmp/test.md - # echo "
" >> ./tmp/test.md
- #       sudo make test-e2e | tee -a ./tmp/test.md
- #       echo "
" >> ./tmp/test.md - # echo "
" >> ./tmp/test.md - - # sudo make test-e2e - - - name: Comment PR with file - uses: thollander/actions-comment-pull-request@v2 - with: - filePath: ./tmp/test.md - comment_tag: nrt_file - reactions: eyes, rocket - mode: recreate - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - continue-on-error: true - - - name: Check outputs - run: | - echo "id : ${{ steps.nrt_message.outputs.id }}" - echo "body : ${{ steps.nrt_message.outputs.body }}" - echo "html_url : ${{ steps.nrt_message.outputs.html_url }}" - - - name: Exec OpenIM System uninstall - run: | - sudo ./scripts/install/install.sh -u - - - name: gobenchdata publish - uses: bobheadxi/gobenchdata@v1 - with: - PRUNE_COUNT: 30 - GO_TEST_FLAGS: -cpu 1,2 - PUBLISH: true - PUBLISH_BRANCH: gh-pages - env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - continue-on-error: true diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml new file mode 100644 index 0000000000..5341c919d6 --- /dev/null +++ b/.github/workflows/go-build-test.yml @@ -0,0 +1,138 @@ +name: Go Build Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + workflow_dispatch: + +jobs: + go-build: + name: Test with go ${{ matrix.go_version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + contents: write + pull-requests: write + strategy: + matrix: + os: [ubuntu-latest] + go_version: ["1.21.x", "1.22.x"] + + steps: + - name: Checkout Server repository + uses: actions/checkout@v4 + + - name: Set up Go ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Get Server dependencies + run: | + go install github.com/magefile/mage@latest + go mod tidy + go mod download + + - name: Set up infra services + uses: hoverkraft-tech/compose-action@v2.0.1 + # Uncomment and set the correct path to your docker-compose file + with: + compose-file: "./docker-compose.yml" + + # run: | + # sudo docker compose up -d + # sudo sleep 30 # Increased sleep time for better stability + # timeout-minutes: 60 # Increased timeout for Docker setup + + + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" + + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml + + - name: Build and test Server Services + run: | + mage build + mage start + mage check + + - name: Checkout Chat repository + uses: actions/checkout@v4 + with: + repository: "openimsdk/chat" + path: "chat-repo" + + - name: Get Chat dependencies + run: | + cd ${{ github.workspace }}/chat-repo + go mod tidy + go mod download + go install github.com/magefile/mage@latest + + - name: Build and test Chat Services + run: | + cd ${{ github.workspace }}/chat-repo + mage build + mage start + mage check + + dockerfile-test: + name: Build and Test Dockerfile + runs-on: ubuntu-latest + strategy: + matrix: + go_version: ["1.21"] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Go ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Get dependencies + run: | + go mod tidy + go mod download + go install github.com/magefile/mage@latest + + - name: Build Docker Image + run: | + IMAGE_NAME="${{ github.event.repository.name }}-test" + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker build -t $IMAGE_NAME . + + - name: Run Docker Container + run: | + IMAGE_NAME="${{ github.event.repository.name }}-test" + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker run --name $CONTAINER_NAME -d $IMAGE_NAME + docker ps -a + + - name: Test Docker Container Logs + run: | + CONTAINER_NAME="${{ github.event.repository.name }}-container" + docker logs $CONTAINER_NAME + + # - name: Cleanup Docker Container + # run: | + # CONTAINER_NAME="${{ github.event.repository.name }}-container" + # IMAGE_NAME="${{ github.event.repository.name }}-test" + # docker stop $CONTAINER_NAME + # docker rm $CONTAINER_NAME + # docker rmi $IMAGE_NAME diff --git a/.github/workflows/golangci-lint.bak b/.github/workflows/golangci-lint.bak deleted file mode 100644 index 64bd498c54..0000000000 --- a/.github/workflows/golangci-lint.bak +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -name: OpenIM golangci-lint -on: - push: - branches: [main] - pull_request: -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: false - - name: OpenIM Scripts Verification(make verify) - run: | - cd scripts - for script in verify-*; do - if [ -x "$script" ]; then - ./"$script" - fi - done - - name: golangci-lint - uses: golangci/golangci-lint-action@v4.0.0 - with: - # Require: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.54 - - # Optional: working directory, useful for monorepos - # working-directory: server - - # Optional: golangci-lint command line arguments. - # - # Note: by default the `.golangci.yml` file should be at the root of the repository. - # The location of the configuration file can be changed by using `--config=` - # args: --timeout=30m --config=/scripts/.golangci.yml --issues-exit-code=0 - - # Optional: show only new issues if it's a pull request. The default value is `false`. - only-new-issues: true - - # Optional:The mode to install golangci-lint. It can be 'binary' or 'goinstall'. - # install-mode: "goinstall" diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml deleted file mode 100644 index b99330c05c..0000000000 --- a/.github/workflows/gosec.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Run Gosec - -# gosec is a source code security audit tool for the Go language. It performs a static -# analysis of the Go code, looking for potential security problems. The main functions of gosec are: -# 1. Find common security vulnerabilities, such as SQL injection, command injection, and cross-site scripting (XSS). -# 2. Audit codes according to common security standards and find non-standard codes. -# 3. Assist the Go language engineer to write safe and reliable code. -# https://github.com/securego/gosec/ -on: - push: - branches: "*" - pull_request: - branches: "*" - paths-ignore: - - '*.md' - - '*.yml' - - '.github' - -jobs: - golang-security-action: - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: ./... - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/help-comment-issue.yml b/.github/workflows/help-comment-issue.yml index c4e72ffc67..b1cc621828 100644 --- a/.github/workflows/help-comment-issue.yml +++ b/.github/workflows/help-comment-issue.yml @@ -29,7 +29,7 @@ jobs: uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.issue.number }} - token: ${{ secrets.BOT_GITHUB_TOKEN }} + token: ${{ secrets.BOT_TOKEN }} body: | This issue is available for anyone to work on. **Make sure to reference this issue in your pull request.** :sparkles: Thank you for your contribution! :sparkles: [Join slack πŸ€–](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) to connect and communicate with our developers. diff --git a/.github/workflows/issue-robot.yml b/.github/workflows/issue-robot.yml deleted file mode 100644 index 2a956ed19c..0000000000 --- a/.github/workflows/issue-robot.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Issue Aotu Translator -on: - issue_comment: - types: [created] - issues: - types: [opened] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: usthe/issues-translate-action@v2.7 - with: - # it is not necessary to decide whether you need to modify the issue header content - IS_MODIFY_TITLE: true - BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - # Required, input your bot github token \ No newline at end of file diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml new file mode 100644 index 0000000000..6a8528ae62 --- /dev/null +++ b/.github/workflows/issue-translator.yml @@ -0,0 +1,19 @@ +name: 'issue-translator' +on: + issue_comment: + types: [created] + issues: + types: [opened] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: usthe/issues-translate-action@v2.7 + with: + BOT_GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} + IS_MODIFY_TITLE: true + # not require, default false, . Decide whether to modify the issue title + # if true, the robot account @Issues-translate-bot must have modification permissions, invite @Issues-translate-bot to your project or use your custom bot. + CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically. πŸ‘―πŸ‘­πŸ»πŸ§‘β€πŸ€β€πŸ§‘πŸ‘«πŸ§‘πŸΏβ€πŸ€β€πŸ§‘πŸ»πŸ‘©πŸΎβ€πŸ€β€πŸ‘¨πŸΏπŸ‘¬πŸΏ + # not require. Customize the translation robot prefix message. \ No newline at end of file diff --git a/.github/workflows/link-pr.yml b/.github/workflows/link-pr.yml deleted file mode 100644 index 6c5ff6c5b9..0000000000 --- a/.github/workflows/link-pr.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Rebot for Link check error - -# Every Monday at 12:30 p.m -on: - schedule: - - cron: '30 12 * * 1' - -jobs: - linkChecker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Link Checker - id: lychee - uses: lycheeverse/lychee-action@v1.9.3 - with: - # For parameter description, see https://github.com/lycheeverse/lychee#commandline-parameters - # Actions Link address -> https://github.com/lycheeverse/lychee-action - # -E, --exclude-all-private Exclude all private IPs from checking. - # -i, --insecure Proceed for server connections considered insecure (invalid TLS) - # -n, --no-progress Do not show progress bar. - # -t, --timeout Website timeout in seconds from connect to response finished [default:20] - # --max-concurrency Maximum number of concurrent network requests [default: 128] - # -a --accept Comma-separated list of accepted status codes for valid links - # docs/.vitepress/dist the site directory to check - # ./*.md all markdown files in the root directory - args: --verbose -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md' - env: - GITHUB_TOKEN: ${{secrets.BOT_GITHUB_TOKEN}} - - - name: Create Issue From File - if: env.lychee_exit_code != 0 - uses: peter-evans/create-issue-from-file@v5 - with: - title: Bug reports for links in OpenIM docs - content-filepath: ./lychee/out.md - labels: kind/documentation, triage/unresolved, report - token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/lock-issue.bak b/.github/workflows/lock-issue.bak deleted file mode 100644 index edf2809653..0000000000 --- a/.github/workflows/lock-issue.bak +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: 'Lock Threads' - -on: - schedule: - - cron: '0 * * * *' - workflow_dispatch: - -permissions: - issues: write - pull-requests: write - -concurrency: - group: lock - -jobs: - action: - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v5 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - issue-inactive-days: '365' - exclude-issue-created-before: '' - exclude-issue-created-after: '' - exclude-issue-created-between: '' - exclude-issue-closed-before: '' - exclude-issue-closed-after: '' - exclude-issue-closed-between: '' - include-any-issue-labels: '' - include-all-issue-labels: '' - exclude-any-issue-labels: '' - add-issue-labels: '' - remove-issue-labels: '' - issue-comment: '' - issue-lock-reason: 'resolved' - pr-inactive-days: '365' - exclude-pr-created-before: '' - exclude-pr-created-after: '' - exclude-pr-created-between: '' - exclude-pr-closed-before: '' - exclude-pr-closed-after: '' - exclude-pr-closed-between: '' - include-any-pr-labels: '' - include-all-pr-labels: '' - exclude-any-pr-labels: '' - add-pr-labels: '' - remove-pr-labels: '' - pr-comment: '' - pr-lock-reason: 'resolved' - process-only: '' - log-output: false \ No newline at end of file diff --git a/.github/workflows/milestone.yml b/.github/workflows/milestone.yml deleted file mode 100644 index c74e7074a7..0000000000 --- a/.github/workflows/milestone.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# shamelessly copied from https://github.com/sigstore/cosign/blob/main/.github/workflows/milestone.yaml - -name: milestone - -on: - pull_request_target: - types: [closed] - branches: - - main - -jobs: - milestone: - runs-on: ubuntu-latest - - permissions: - actions: none - checks: none - contents: read - deployments: none - issues: write - packages: none - pull-requests: write - repository-projects: none - security-events: none - statuses: none - - steps: - - uses: actions/github-script@v7 # v6 - with: - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} - script: | - if (!context.payload.pull_request.merged) { - console.log('PR was not merged, skipping.'); - return; - } - - if (!!context.payload.pull_request.milestone) { - console.log('PR has existing milestone, skipping.'); - return; - } - - milestones = await github.rest.issues.listMilestones({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - sort: 'title', - direction: 'desc' - }) - - if (milestones.data.length === 0) { - console.log('There are no milestones, skipping.'); - return; - } - - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - milestone: milestones.data[0].number - }); diff --git a/.github/workflows/opencommit.yml b/.github/workflows/opencommit.yml deleted file mode 100644 index d483ef1f65..0000000000 --- a/.github/workflows/opencommit.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM OpenCommit Action - -on: - push: - # this list of branches is often enough, - # but you may still ignore other public branches - branches-ignore: [main master dev development release] - -jobs: - opencommit: - timeout-minutes: 10 - name: OpenCommit - runs-on: ubuntu-latest - permissions: write-all - steps: - - name: Setup Node.js Environment - uses: actions/setup-node@v4 - with: - node-version: '16' - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: di-sukharev/opencommit@github-action-v1.0.4 - with: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - - env: - # set openAI api key in repo actions secrets, - # for openAI keys go to: https://platform.openai.com/account/api-keys - # for repo secret go to: /settings/secrets/actions - OCO_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - # customization - OCO_OPENAI_MAX_TOKENS: 500 - OCO_OPENAI_BASE_PATH: '' - OCO_DESCRIPTION: false - OCO_EMOJI: false - OCO_MODEL: gpt-3.5-turbo-16k - OCO_LANGUAGE: en - OCO_PROMPT_MODULE: conventional-commit - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml deleted file mode 100644 index 83d495a0e4..0000000000 --- a/.github/workflows/openimci.yml +++ /dev/null @@ -1,191 +0,0 @@ - -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: OpenIM CI Auto Build - -on: - push: - branches: - - main - - release-* - paths-ignore: - - "docs/**" - - "README.md" - - "README_zh-CN.md" - - "**.md" - - "docs/**" - - "CONTRIBUTING.md" - pull_request: - branches: - - main - - release-* - paths-ignore: - - "README.md" - - "README_zh-CN.md" - - "CONTRIBUTING/**" - - "**.md" - - "docs/**" - workflow_dispatch: - -jobs: - - build-linux: - name: Execute OpenIM Script On Linux - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - environment: - name: openim - strategy: - matrix: - arch: [arm64, armv7, amd64] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - - name: Set up Docker for Linux - run: | - sudo docker compose up -d - sudo sleep 30 # Increased sleep time for better stability - timeout-minutes: 20 # Increased timeout for Docker setup - - - - name: init - run: sudo bash bootstrap.sh - timeout-minutes: 20 - - - name: Build, Start, Check Services and Print Logs for Linux - run: | - sudo mage - sudo mage start - sudo mage check - - - - name: Restart Services and Print Logs - run: | - sudo mage stop - sudo mage start - sudo mage check - - -# build-mac: -# name: Execute OpenIM Script On macOS -# runs-on: macos-latest -# permissions: -# contents: write -# pull-requests: write -# environment: -# name: openim -# strategy: -# matrix: -# arch: [arm64, armv7, amd64] -# -# steps: -# - uses: actions/checkout@v3 - -# - name: Set up Go -# uses: actions/setup-go@v4 -# with: -# go-version: '1.21' - - -# while ! docker system info > /dev/null 2>&1; do -# echo "Waiting for Docker to start..." -# sleep 10 # Increased delay to ensure Docker starts properly -# done - -# - name: Install Docker -# run: | -# brew install docker -# brew install docker-compose -# sleep 10 -# docker-compose up -d -# sleep 30 -# timeout-minutes: 20 -# - -# - name: init -# run: sudo bash bootstrap.sh -# timeout-minutes: 20 - -# - name: Build, Start, Check Services and Print Logs for Linux -# run: | -# sudo mage -# sudo mage start -# sudo mage check - -# - name: Restart Services and Print Logs -# run: | -# sudo mage stop -# sudo mage start -# sudo mage check - -# build-windows: -# name: Execute OpenIM Script On Windows -# runs-on: windows-latest -# permissions: -# contents: write -# pull-requests: write -# environment: -# name: openim -# strategy: -# matrix: -# arch: [arm64, armv7, amd64] -# -# steps: -# - uses: actions/checkout@v3 - -# - name: Set up Go -# uses: actions/setup-go@v4 -# with: -# go-version: '1.21' - -# - name: Set up Docker for Windows -# run: | -# $images = @("zookeeper", "redis", "kafka") -# foreach ($image in $images) { -# $tag = "$image:latest" -# docker pull $tag | Out-Null -# if ($LASTEXITCODE -ne 0) { -# Write-Host "Skipping $image as it is not available for Windows" -# } else { -# Write-Host "Successfully pulled $image" -# } -# } -# docker compose up -d -# Start-Sleep -Seconds 30 -# timeout-minutes: 20 -# shell: pwsh - -# - name: init -# run: bootstrap.bat -# timeout-minutes: 20 - -# - name: Build, Start, Check Services and Print Logs for Linux -# run: | -# mage -# mage start -# mage check - -# - name: Restart Services and Print Logs -# run: | -# mage stop -# mage start -# mage check diff --git a/.github/workflows/project-progress.yml b/.github/workflows/project-progress.yml deleted file mode 100644 index 87a4c13810..0000000000 --- a/.github/workflows/project-progress.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Β© 2023 OpenIM open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GitHub recommends pinning actions to a commit SHA. -# To get a newer version, you will need to update the SHA. -# You can also reference a tag or branch, but the action may change without warning. - -name: Move assigned card -on: - issues: - types: - - assigned - pull_request: - types: - - assigned - branches-ignore: - - 'asf-auto-updates' - - 'ignore' - -jobs: - move-assigned-card: - runs-on: ubuntu-latest - steps: - - uses: alex-page/github-project-automation-plus@v0.9.0 - with: - project: openim-powerful - column: In Progress - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml new file mode 100644 index 0000000000..7d7d23f173 --- /dev/null +++ b/.github/workflows/publish-docker-image.yml @@ -0,0 +1,145 @@ +name: Publish Docker image to registries + +on: + push: + branches: + - release-* + + release: + types: [published] + + workflow_dispatch: + inputs: + tag: + description: "Tag version to be used for Docker image" + required: true + default: "v3.8.0" + +# env: +# GO_VERSION: "1.21" + +jobs: + publish-docker-images: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: main-repo + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./main-repo + load: true + tags: "openim/openim-server:local" + + - name: Checkout compose repository + uses: actions/checkout@v4 + with: + repository: "openimsdk/openim-docker" + path: "compose-repo" + + - name: Get Internal IP Address + id: get-ip + run: | + IP=$(hostname -I | awk '{print $1}') + echo "The IP Address is: $IP" + echo "::set-output name=ip::$IP" + + - name: Update .env to use the local image + run: | + sed -i 's|OPENIM_SERVER_IMAGE=.*|OPENIM_SERVER_IMAGE=openim/openim-server:local|' ${{ github.workspace }}/compose-repo/.env + sed -i 's|MINIO_EXTERNAL_ADDRESS=.*|MINIO_EXTERNAL_ADDRESS=http://${{ steps.get-ip.outputs.ip }}:10005|' ${{ github.workspace }}/compose-repo/.env + + - name: Start services using Docker Compose + run: | + cd ${{ github.workspace }}/compose-repo + docker compose up -d + sleep 60 + + - name: Check openim-server health + run: | + timeout=300 + interval=30 + elapsed=0 + while [[ $elapsed -le $timeout ]]; do + if ! docker exec openim-server mage check; then + echo "openim-server is not ready, waiting..." + sleep $interval + elapsed=$(($elapsed + $interval)) + else + echo "Health check successful" + exit 0 + fi + done + echo "Health check failed after 5 minutes" + exit 1 + + - name: Check openim-chat health + if: success() + run: | + if ! docker exec openim-chat mage check; then + echo "openim-chat check failed" + exit 1 + else + echo "Health check successful" + exit 0 + fi + + + - name: Extract metadata for Docker # (tags, labels) + if: success() + id: meta + uses: docker/metadata-action@v5.5.1 + with: + images: | + openim/openim-server + ghcr.io/openimsdk/openim-server + registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server + + # generate Docker tags based on the following events/attributes + tags: | + type=ref,event=tag + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern=v{{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Aliyun Container Registry + uses: docker/login-action@v2 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.ALIREGISTRY_USERNAME }} + password: ${{ secrets.ALIREGISTRY_TOKEN }} + + - name: Build and push Docker images + uses: docker/build-push-action@v5 + with: + context: ./main-repo + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/pull-request.bak b/.github/workflows/pull-request.bak deleted file mode 100644 index f7c5900ce7..0000000000 --- a/.github/workflows/pull-request.bak +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Github Pull Request -on: - workflow_dispatch: - schedule: - - cron: '0 2 * * *' - -permissions: - contents: write - pull-requests: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v4 - - name: Setup Go - uses: actions/setup-go@v5 - - name: Run go modules tidy - run: | - sudo apt-get install jq - sudo make tidy - sudo make tools.verify.go-gitlint - echo "Run go modules tidy successfully" - continue-on-error: true - - - name: Run go format and lint - run: | - sudo make format - echo "Run go format successfully" - continue-on-error: true - - - name: Run go lint - run: | - sudo make lint - echo "Run go lint successfully" - continue-on-error: true - - - name: Generate all necessary files, such as error code files - run: | - make gen.docgo.doc - make gen - echo "Generate all necessary files successfully" - continue-on-error: true - - - name: make init - run: | - export OPENIM_IP=127.0.0.1 - export LOG_STORAGE_LOCATION="../logs/" - ./scripts/init-config.sh --examples --force - echo "Generate all necessary files successfully" - continue-on-error: true - - - name: Generate Versions Including Pre-release Identifiers - run: | - latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`) - echo $latest_tag > pkg/common/config/version - continue-on-error: true - - - name: Gen CHANGELOG file - run: | - current_tag=$(git describe --tags --abbrev=0) - version=$(echo "$current_tag" | sed -E 's/^v?([0-9]+)\.([0-9]+)\..*$/\1.\2/') - echo "OpenIM Version: $version" - make tools.install.git-chglog - cd CHANGELOG - git-chglog --tag-filter-pattern "v${version}.*" -o CHANGELOG-${version}.md - cd .. - continue-on-error: true - - - name: Run unit test and get test coverage - run: | - make cover - echo "Run unit test and get test coverage successfully" - continue-on-error: true - - - name: OpenIM verify copyright - run: | - sudo make add-copyright - echo "OpenIM verify successfully" - continue-on-error: true - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.BOT_GITHUB_TOKEN }} - commit-message: "cicd: bump League Patch" - author: kubbot <3293172751ysy@gmail.com> - committer: kubbot <3293172751ysy@gmail.com> - # signoff: false - # draft: false - branch: "asf-auto-updates" - assignees: cubxxw - reviewers: cubxxw - title: "[Auto PR πŸ€–] Bump League Patch auto PR" - body: | - I am a PR generated by robot automation. - - Review criteria: - - - [ ] Disenchanter can connect and issue actions - - Github Actions Status: - - [![Github Pull Request](https://github.com/openimsdk/open-im-server/actions/workflows/pull-request.yml/badge.svg)](https://github.com/openimsdk/open-im-server/actions/workflows/pull-request.yml) - - This is an automated PR. - [workflow](https://github.com/openimsdk/open-im-server/blob/main/.github/workflows/pull-request.yml). - labels: | - kind/documentation - enhancement - report diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index 251f558764..0000000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Release Drafter - -on: - push: - # branches to consider in the event; optional, defaults to all - branches: - - main - # pull_request event is required only for autolabeler - pull_request: - # Only following types are handled by the action, but one can default to all as well - # types: [opened, reopened, synchronize] - # pull_request_target event is required for autolabeler to support PRs from forks - # pull_request_target: - # types: [opened, reopened, synchronize] - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - # write permission is required to create a github release - contents: write - # write permission is required for autolabeler - # otherwise, read permission is required at least - pull-requests: write - runs-on: ubuntu-latest - steps: - # (Optional) GitHub Enterprise requires GHE_HOST variable set - #- name: Set GHE_HOST - # run: | - # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV - - # Drafts your next Release notes as Pull Requests are merged into "master" - - uses: release-drafter/release-drafter@v6 - # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml - # with: - # config-name: my-config.yml - # disable-autolabeler: true - env: - GITHUB_TOKEN: ${{ secrets.REDBOT_GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.bak b/.github/workflows/release.bak deleted file mode 100644 index c15cff6a3c..0000000000 --- a/.github/workflows/release.bak +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM Server Release Workflow - -on: - push: - # run only against tags - tags: - - '*' - -permissions: - contents: write - packages: write - issues: write - -jobs: - goreleaser: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - uses: actions/setup-go@v5 - with: - go-version: stable - # More assembly might be required: Docker logins, GPG, etc. It all depends - # on your needs. - - uses: goreleaser/goreleaser-action@v5 - with: - # either 'goreleaser' (default) or 'goreleaser-pro': - distribution: goreleaser - version: latest - workdir: . - args: release -f ./build/goreleaser.yaml --clean --release-footer-tmpl=scripts/template/footer.md.tmpl --release-header-tmpl=scripts/template/head.md.tmpl - env: - USERNAME: ${{ github.repository_owner }} - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - FURY_TOKEN: ${{ secrets.FURY_TOKEN }} - # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' - # distribution: - # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - - goreleaser-check-pkgs: - runs-on: ubuntu-latest - env: - DOCKER_CLI_EXPERIMENTAL: "enabled" - needs: [ goreleaser ] - if: github.ref == 'refs/heads/main' - strategy: - matrix: - format: [ deb, rpm, apk ] - steps: - - uses: actions/checkout@v4 # v3 - with: - fetch-depth: 0 - - uses: arduino/setup-task@e26d8975574116b0097a1161e0fe16ba75d84c1c # v1 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/setup-qemu-action@326560df218a7ea9cf6ab49bbc88b8b306bb437e # v2 - - uses: actions/cache@a2ed59d39b352305bdd2f628719a53b2cc4f9613 # v3 - with: - path: | - ./_output/dist/*.deb - ./_output/dist/*.rpm - ./_output/dist/*.apk - key: ${{ github.ref }} - - run: task goreleaser:test:${{ matrix.format }} diff --git a/.github/workflows/remove-unused-labels.yml b/.github/workflows/remove-unused-labels.yml new file mode 100644 index 0000000000..ab80b1f96e --- /dev/null +++ b/.github/workflows/remove-unused-labels.yml @@ -0,0 +1,74 @@ +name: Remove Unused Labels +on: + workflow_dispatch: + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + contents: read + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Fetch All Issues and PRs + id: fetch_issues_prs + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'all', + per_page: 100 + }); + + const labelsInUse = new Set(); + issues.forEach(issue => { + issue.labels.forEach(label => { + labelsInUse.add(label.name); + }); + }); + + return JSON.stringify(Array.from(labelsInUse)); + result-encoding: string + + - name: Fetch All Labels + id: fetch_labels + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labels = await github.paginate(github.rest.issues.listLabelsForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100 + }); + + return JSON.stringify(labels.map(label => label.name)); + result-encoding: string + + - name: Remove Unused Labels + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelsInUse = new Set(JSON.parse(process.env.LABELS_IN_USE)); + const allLabels = JSON.parse(process.env.ALL_LABELS); + + const unusedLabels = allLabels.filter(label => !labelsInUse.has(label)); + + for (const label of unusedLabels) { + await github.rest.issues.deleteLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label + }); + console.log(`Deleted label: ${label}`); + } + env: + LABELS_IN_USE: ${{ steps.fetch_issues_prs.outputs.result }} + ALL_LABELS: ${{ steps.fetch_labels.outputs.result }} diff --git a/.github/workflows/reopen-issue.yml b/.github/workflows/reopen-issue.yml new file mode 100644 index 0000000000..32f838ba48 --- /dev/null +++ b/.github/workflows/reopen-issue.yml @@ -0,0 +1,78 @@ +name: Reopen and Update Stale Issues + +on: + workflow_dispatch: + +jobs: + reopen_stale_issues: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Fetch Closed Issues with lifecycle/stale Label + id: fetch_issues + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed', + labels: 'lifecycle/stale', + per_page: 100 + }); + const issueNumbers = issues + .filter(issue => !issue.pull_request) // exclude PR + .map(issue => issue.number); + console.log(`Fetched issues: ${issueNumbers}`); + return issueNumbers; + + - name: Set issue numbers + id: set_issue_numbers + run: | + echo "ISSUE_NUMBERS=${{ steps.fetch_issues.outputs.result }}" >> $GITHUB_ENV + echo "Issue numbers: ${{ steps.fetch_issues.outputs.result }}" + + - name: Reopen Issues + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issueNumbers = JSON.parse(process.env.ISSUE_NUMBERS); + console.log(`Reopening issues: ${issueNumbers}`); + + for (const issue_number of issueNumbers) { + // Reopen the issue + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + state: 'open' + }); + console.log(`Reopened issue #${issue_number}`); + } + + - name: Remove lifecycle/stale Label + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issueNumbers = JSON.parse(process.env.ISSUE_NUMBERS); + console.log(`Removing 'lifecycle/stale' label from issues: ${issueNumbers}`); + + for (const issue_number of issueNumbers) { + // Remove the lifecycle/stale label + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + name: 'lifecycle/stale' + }); + console.log(`Removed label 'lifecycle/stale' from issue #${issue_number}`); + } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 4445f6ddaa..0000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. -# -# You can adjust the behavior by modifying this file. -# For more information, see: -# https://github.com/actions/stale -name: Mark stale issues and pull requests - -on: - schedule: - - cron: '0 8 * * 1' - -jobs: - stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v9 - with: - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} - days-before-stale: 60 - days-before-close: 305 - stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days.' - stale-pr-message: 'This issue is stale because it has been open 60 days with no activity.' - close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.' - close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity. You can reopen it if you want.' - stale-pr-label: lifecycle/stale - stale-issue-label: lifecycle/stale - exempt-issue-labels: 'openim' - exempt-pr-labels: 'openim' - exempt-draft-pr: true diff --git a/.github/workflows/sync-release.bak b/.github/workflows/sync-release.bak deleted file mode 100644 index a85c74fde0..0000000000 --- a/.github/workflows/sync-release.bak +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright Β© 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -name: Synchronize OpenIM Release Branch Public Code To Other Repositories -on: - push: - paths: - - scripts/* - - docs/* - - config/* - branches: - - release-v*.* - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run GitHub File Sync - uses: BetaHuhn/repo-file-sync-action@latest - with: - GH_INSTALLATION_TOKEN: "${{ secrets.BOT_GITHUB_TOKEN }}" - CONFIG_PATH: .github/sync-release.yml - ORIGINAL_MESSAGE: true - SKIP_PR: true - COMMIT_EACH_FILE: false - COMMIT_BODY: "πŸ€– kubbot to synchronize the warehouse" - GIT_EMAIL: "3293172751ysy@gmail.com" - GIT_USERNAME: "kubbot" - PR_BODY: πŸ‘Œ kubecub provides automated community services - REVIEWERS: | - kubbot - cubxxw - PR_LABELS: | - file-sync - automerge - ASSIGNEES: | - kubbot - continue-on-error: true diff --git a/.github/workflows/sync.bak b/.github/workflows/sync.bak deleted file mode 100644 index 595cbbe2c8..0000000000 --- a/.github/workflows/sync.bak +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright Β© 2023 KubeCub open source community. All rights reserved. -# Licensed under the MIT License (the "License"); -# you may not use this file except in compliance with the License. - -# https://github.com/BetaHuhn/repo-file-sync-action -name: Synchronize OpenIM Main Branch Public Code To Other Repositories -on: - push: - branches: - - main - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run GitHub File Sync - uses: BetaHuhn/repo-file-sync-action@latest - with: - GH_INSTALLATION_TOKEN: "${{ secrets.BOT_GITHUB_TOKEN }}" - CONFIG_PATH: .github/sync.yml - ORIGINAL_MESSAGE: true - SKIP_PR: true - COMMIT_EACH_FILE: false - COMMIT_BODY: "πŸ€– kubbot to synchronize the warehouse" - GIT_EMAIL: "3293172751ysy@gmail.com" - GIT_USERNAME: "kubbot" - PR_BODY: πŸ‘Œ kubecub provides automated community services - REVIEWERS: | - kubbot - cubxxw - PR_LABELS: | - file-sync - automerge - ASSIGNEES: | - kubbot - continue-on-error: true diff --git a/.github/workflows/greetings.yml b/.github/workflows/user-first-interaction.yml similarity index 65% rename from .github/workflows/greetings.yml rename to .github/workflows/user-first-interaction.yml index b1c85ee375..6999889eb0 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/user-first-interaction.yml @@ -1,18 +1,4 @@ -# Copyright Β© 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: OpenIM First Interaction +name: User First Interaction on: issues: @@ -28,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/first-interaction@v1.3.0 with: - repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} + repo-token: ${{ secrets.BOT_TOKEN }} pr-message: | Hello! Thank you for your contribution. diff --git a/.gitignore b/.gitignore index fb8d428d24..77cf855b70 100644 --- a/.gitignore +++ b/.gitignore @@ -34,11 +34,7 @@ deployments/charts/generated-configs/ ### OpenIM Config ### .env config/config.yaml -config/alertmanager.yml -config/prometheus.yml -config/email.tmpl config/notification.yaml -config/instance-down-rules.yml ### OpenIM deploy ### deployments/openim-server/charts diff --git a/.golangci.yml b/.golangci.yml index ae8cea6732..a95e980f85 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,20 +1,3 @@ -# Copyright Β© 2023 OpenIMSDK open source community. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file contains all available configuration options -# with their default values. - # options for analysis running run: # default concurrency is a available CPU number @@ -302,7 +285,7 @@ linters-settings: gofumpt: # Select the Go version to target. The default is `1.18`. - lang-version: "1.20" + go-version: "1.21" # Choose whether or not to use the extra rules that are disabled # by default diff --git a/CHANGELOG/CHANGELOG-1.0.md b/CHANGELOG/CHANGELOG-1.0.md deleted file mode 100644 index bed425943c..0000000000 --- a/CHANGELOG/CHANGELOG-1.0.md +++ /dev/null @@ -1,56 +0,0 @@ -# Version logging for OpenIM - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v1.0.7 - 2021-12-17](#v107---2021-12-17) - - [v1.0.6 - 2021-12-10](#v106---2021-12-10) - - [v1.0.5 - 2021-12-03](#v105---2021-12-03) - - [v1.0.4 - 2021-11-25](#v104---2021-11-25) - - [v1.0.3 - 2021-11-12](#v103---2021-11-12) - - [v1.0.1 - 2021-11-04](#v101---2021-11-04) - - [v1.0.0 - 2021-10-28](#v100---2021-10-28) - - [Reverts](#reverts) - - - - -## [Unreleased] - - - -## [v1.0.7] - 2021-12-17 - - -## [v1.0.6] - 2021-12-10 - - -## [v1.0.5] - 2021-12-03 - - -## [v1.0.4] - 2021-11-25 - - -## [v1.0.3] - 2021-11-12 - - -## [v1.0.1] - 2021-11-04 - - -## v1.0.0 - 2021-10-28 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v1.0.7...HEAD -[v1.0.7]: https://github.com/openimsdk/open-im-server/compare/v1.0.6...v1.0.7 -[v1.0.6]: https://github.com/openimsdk/open-im-server/compare/v1.0.5...v1.0.6 -[v1.0.5]: https://github.com/openimsdk/open-im-server/compare/v1.0.4...v1.0.5 -[v1.0.4]: https://github.com/openimsdk/open-im-server/compare/v1.0.3...v1.0.4 -[v1.0.3]: https://github.com/openimsdk/open-im-server/compare/v1.0.1...v1.0.3 -[v1.0.1]: https://github.com/openimsdk/open-im-server/compare/v1.0.0...v1.0.1 diff --git a/CHANGELOG/CHANGELOG-2.0.md b/CHANGELOG/CHANGELOG-2.0.md deleted file mode 100644 index 8902f18147..0000000000 --- a/CHANGELOG/CHANGELOG-2.0.md +++ /dev/null @@ -1,87 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v2.0.10] - 2022-05-13 - - -## [v2.0.9] - 2022-04-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.8] - 2022-04-24 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.7] - 2022-04-08 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.6] - 2022-04-01 -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.0.5] - 2022-03-24 - - -## [v2.04] - 2022-03-18 - - -## [v2.0.3] - 2022-03-11 - - -## [v2.0.2] - 2022-03-04 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.1] - 2022-02-25 - - -## v2.0.0 - 2022-02-23 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.0.10...HEAD -[v2.0.10]: https://github.com/openimsdk/open-im-server/compare/v2.0.9...v2.0.10 -[v2.0.9]: https://github.com/openimsdk/open-im-server/compare/v2.0.8...v2.0.9 -[v2.0.8]: https://github.com/openimsdk/open-im-server/compare/v2.0.7...v2.0.8 -[v2.0.7]: https://github.com/openimsdk/open-im-server/compare/v2.0.6...v2.0.7 -[v2.0.6]: https://github.com/openimsdk/open-im-server/compare/v2.0.5...v2.0.6 -[v2.0.5]: https://github.com/openimsdk/open-im-server/compare/v2.04...v2.0.5 -[v2.04]: https://github.com/openimsdk/open-im-server/compare/v2.0.3...v2.04 -[v2.0.3]: https://github.com/openimsdk/open-im-server/compare/v2.0.2...v2.0.3 -[v2.0.2]: https://github.com/openimsdk/open-im-server/compare/v2.0.1...v2.0.2 -[v2.0.1]: https://github.com/openimsdk/open-im-server/compare/v2.0.0...v2.0.1 diff --git a/CHANGELOG/CHANGELOG-2.1.md b/CHANGELOG/CHANGELOG-2.1.md deleted file mode 100644 index 7eef06f08a..0000000000 --- a/CHANGELOG/CHANGELOG-2.1.md +++ /dev/null @@ -1,50 +0,0 @@ -# Version logging for OpenIM:v2.1 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.1](#version-logging-for-openimv21) - - [Unreleased](#unreleased) - - [v2.1.0 - 2022-06-17](#v210---2022-06-17) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## v2.1.0 - 2022-06-17 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.1.0...HEAD diff --git a/CHANGELOG/CHANGELOG-2.2.md b/CHANGELOG/CHANGELOG-2.2.md deleted file mode 100644 index 8afb24b29f..0000000000 --- a/CHANGELOG/CHANGELOG-2.2.md +++ /dev/null @@ -1,50 +0,0 @@ -# Version logging for OpenIM:v2.2 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.2](#version-logging-for-openimv22) - - [Unreleased](#unreleased) - - [v2.2.0 - 2022-07-01](#v220---2022-07-01) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## v2.2.0 - 2022-07-01 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.2.0...HEAD diff --git a/CHANGELOG/CHANGELOG-2.3.md b/CHANGELOG/CHANGELOG-2.3.md deleted file mode 100644 index e14745f30d..0000000000 --- a/CHANGELOG/CHANGELOG-2.3.md +++ /dev/null @@ -1,70 +0,0 @@ -# Version logging for OpenIM:v2.3 - -> **Note**: -> Deprecated version logging for OpenIM, please refer to [CHANGELOG.md](../CHANGELOG.md) - - - -- [Version logging for OpenIM:v2.3](#version-logging-for-openimv23) - - [Unreleased](#unreleased) - - [v2.3.3 - 2022-09-18](#v233---2022-09-18) - - [v2.3.2 - 2022-09-09](#v232---2022-09-09) - - [v2.3.0-rc2 - 2022-07-29](#v230-rc2---2022-07-29) - - [v2.3.0-rc1 - 2022-07-25](#v230-rc1---2022-07-25) - - [v2.3.0-rc0 - 2022-07-15](#v230-rc0---2022-07-15) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## [v2.3.3] - 2022-09-18 - - -## [v2.3.2] - 2022-09-09 - - -## [v2.3.0-rc2] - 2022-07-29 - - -## [v2.3.0-rc1] - 2022-07-25 - - -## v2.3.0-rc0 - 2022-07-15 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) -- friend modify -- update - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.3.3...HEAD -[v2.3.3]: https://github.com/openimsdk/open-im-server/compare/v2.3.2...v2.3.3 -[v2.3.2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc2...v2.3.2 -[v2.3.0-rc2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc1...v2.3.0-rc2 -[v2.3.0-rc1]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc0...v2.3.0-rc1 diff --git a/CHANGELOG/CHANGELOG-2.9.md b/CHANGELOG/CHANGELOG-2.9.md deleted file mode 100644 index b921eb51b8..0000000000 --- a/CHANGELOG/CHANGELOG-2.9.md +++ /dev/null @@ -1,52 +0,0 @@ -# Version logging for OpenIM - - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v2.9.0+1.839643f - 2023-07-07](#v2901839643f---2023-07-07) - - [v2.9.0+2.35f07fe - 2023-07-06](#v290235f07fe---2023-07-06) - - [v2.9.0+1.b5072b1 - 2023-07-05](#v2901b5072b1---2023-07-05) - - [v2.9.0+3.2667a3a - 2023-07-05](#v29032667a3a---2023-07-05) - - [v2.9.0+7.04818ca - 2023-07-05](#v290704818ca---2023-07-05) - - [v2.9.0 - 2023-07-04](#v290---2023-07-04) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - - - - -## [Unreleased] - - - -## [v2.9.0+1.839643f] - 2023-07-07 - - -## [v2.9.0+2.35f07fe] - 2023-07-06 - - -## [v2.9.0+1.b5072b1] - 2023-07-05 - - -## [v2.9.0+3.2667a3a] - 2023-07-05 - - -## [v2.9.0+7.04818ca] - 2023-07-05 - - -## v2.9.0 - 2023-07-04 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+1.839643f...HEAD -[v2.9.0+1.839643f]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+2.35f07fe...v2.9.0+1.839643f -[v2.9.0+2.35f07fe]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+1.b5072b1...v2.9.0+2.35f07fe -[v2.9.0+1.b5072b1]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+3.2667a3a...v2.9.0+1.b5072b1 -[v2.9.0+3.2667a3a]: https://github.com/openimsdk/open-im-server/compare/v2.9.0+7.04818ca...v2.9.0+3.2667a3a -[v2.9.0+7.04818ca]: https://github.com/openimsdk/open-im-server/compare/v2.9.0...v2.9.0+7.04818ca diff --git a/CHANGELOG/CHANGELOG-3.0.md b/CHANGELOG/CHANGELOG-3.0.md deleted file mode 100644 index 1b99e7e34a..0000000000 --- a/CHANGELOG/CHANGELOG-3.0.md +++ /dev/null @@ -1,188 +0,0 @@ -# Version logging for OpenIM - - -- [Version logging for OpenIM](#version-logging-for-openim) - - [Unreleased](#unreleased) - - [v2.9.0 - 2023-07-04](#v290---2023-07-04) - - [Reverts](#reverts) - - [Pull Requests](#pull-requests) - - [v2.3.3 - 2022-09-18](#v233---2022-09-18) - - [v2.3.2 - 2022-09-09](#v232---2022-09-09) - - [v2.3.0-rc2 - 2022-07-29](#v230-rc2---2022-07-29) - - [v2.3.0-rc1 - 2022-07-25](#v230-rc1---2022-07-25) - - [v2.3.0-rc0 - 2022-07-15](#v230-rc0---2022-07-15) - - [v2.2.0 - 2022-07-01](#v220---2022-07-01) - - [v2.1.0 - 2022-06-17](#v210---2022-06-17) - - [Pull Requests](#pull-requests-1) - - [v2.0.10 - 2022-05-13](#v2010---2022-05-13) - - [v2.0.9 - 2022-04-29](#v209---2022-04-29) - - [Reverts](#reverts-1) - - [Pull Requests](#pull-requests-2) - - [v2.0.7 - 2022-04-08](#v207---2022-04-08) - - [Pull Requests](#pull-requests-3) - - [v2.0.6 - 2022-04-01](#v206---2022-04-01) - - [Pull Requests](#pull-requests-4) - - [v2.0.5 - 2022-03-24](#v205---2022-03-24) - - [v2.04 - 2022-03-18](#v204---2022-03-18) - - [v2.0.3 - 2022-03-11](#v203---2022-03-11) - - [v2.0.2 - 2022-03-04](#v202---2022-03-04) - - [Pull Requests](#pull-requests-5) - - [v2.0.1 - 2022-02-25](#v201---2022-02-25) - - [v2.0.0 - 2022-02-23](#v200---2022-02-23) - - [v1.0.7 - 2021-12-17](#v107---2021-12-17) - - [v1.0.6 - 2021-12-10](#v106---2021-12-10) - - [v1.0.5 - 2021-12-03](#v105---2021-12-03) - - [v1.0.4 - 2021-11-25](#v104---2021-11-25) - - [v1.0.3 - 2021-11-12](#v103---2021-11-12) - - [v1.0.1 - 2021-11-04](#v101---2021-11-04) - - [v1.0.0 - 2021-10-28](#v100---2021-10-28) - - [Reverts](#reverts-2) - - - - -## [Unreleased] - - - -## [v2.9.0] - 2023-07-04 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.3.3] - 2022-09-18 - - -## [v2.3.2] - 2022-09-09 - - -## [v2.3.0-rc2] - 2022-07-29 - - -## [v2.3.0-rc1] - 2022-07-25 - - -## [v2.3.0-rc0] - 2022-07-15 - - -## [v2.2.0] - 2022-07-01 - - -## [v2.1.0] - 2022-06-17 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.10] - 2022-05-13 - - -## [v2.0.9] - 2022-04-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.7] - 2022-04-08 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.6] - 2022-04-01 -### Pull Requests -- Merge branch 'tuoyun' - - - -## [v2.0.5] - 2022-03-24 - - -## [v2.04] - 2022-03-18 - - -## [v2.0.3] - 2022-03-11 - - -## [v2.0.2] - 2022-03-04 -### Pull Requests -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' -- Merge branch 'tuoyun' - - - -## [v2.0.1] - 2022-02-25 - - -## [v2.0.0] - 2022-02-23 - - -## [v1.0.7] - 2021-12-17 - - -## [v1.0.6] - 2021-12-10 - - -## [v1.0.5] - 2021-12-03 - - -## [v1.0.4] - 2021-11-25 - - -## [v1.0.3] - 2021-11-12 - - -## [v1.0.1] - 2021-11-04 - - -## v1.0.0 - 2021-10-28 -### Reverts -- friend modify -- update - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v2.9.0...HEAD -[v2.9.0]: https://github.com/openimsdk/open-im-server/compare/v2.3.3...v2.9.0 -[v2.3.3]: https://github.com/openimsdk/open-im-server/compare/v2.3.2...v2.3.3 -[v2.3.2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc2...v2.3.2 -[v2.3.0-rc2]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc1...v2.3.0-rc2 -[v2.3.0-rc1]: https://github.com/openimsdk/open-im-server/compare/v2.3.0-rc0...v2.3.0-rc1 -[v2.3.0-rc0]: https://github.com/openimsdk/open-im-server/compare/v2.2.0...v2.3.0-rc0 -[v2.2.0]: https://github.com/openimsdk/open-im-server/compare/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/openimsdk/open-im-server/compare/v2.0.10...v2.1.0 -[v2.0.10]: https://github.com/openimsdk/open-im-server/compare/v2.0.9...v2.0.10 -[v2.0.9]: https://github.com/openimsdk/open-im-server/compare/v2.0.7...v2.0.9 -[v2.0.7]: https://github.com/openimsdk/open-im-server/compare/v2.0.6...v2.0.7 -[v2.0.6]: https://github.com/openimsdk/open-im-server/compare/v2.0.5...v2.0.6 -[v2.0.5]: https://github.com/openimsdk/open-im-server/compare/v2.04...v2.0.5 -[v2.04]: https://github.com/openimsdk/open-im-server/compare/v2.0.3...v2.04 -[v2.0.3]: https://github.com/openimsdk/open-im-server/compare/v2.0.2...v2.0.3 -[v2.0.2]: https://github.com/openimsdk/open-im-server/compare/v2.0.1...v2.0.2 -[v2.0.1]: https://github.com/openimsdk/open-im-server/compare/v2.0.0...v2.0.1 -[v2.0.0]: https://github.com/openimsdk/open-im-server/compare/v1.0.7...v2.0.0 -[v1.0.7]: https://github.com/openimsdk/open-im-server/compare/v1.0.6...v1.0.7 -[v1.0.6]: https://github.com/openimsdk/open-im-server/compare/v1.0.5...v1.0.6 -[v1.0.5]: https://github.com/openimsdk/open-im-server/compare/v1.0.4...v1.0.5 -[v1.0.4]: https://github.com/openimsdk/open-im-server/compare/v1.0.3...v1.0.4 -[v1.0.3]: https://github.com/openimsdk/open-im-server/compare/v1.0.1...v1.0.3 -[v1.0.1]: https://github.com/openimsdk/open-im-server/compare/v1.0.0...v1.0.1 diff --git a/CHANGELOG/CHANGELOG-3.1.md b/CHANGELOG/CHANGELOG-3.1.md deleted file mode 100644 index 8944321830..0000000000 --- a/CHANGELOG/CHANGELOG-3.1.md +++ /dev/null @@ -1,20 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## v3.1.0 - 2023-07-28 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.1.0...HEAD diff --git a/CHANGELOG/CHANGELOG-3.2.md b/CHANGELOG/CHANGELOG-3.2.md deleted file mode 100644 index 3f77a82738..0000000000 --- a/CHANGELOG/CHANGELOG-3.2.md +++ /dev/null @@ -1,32 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.2.2-alpha.0] - 2023-08-25 - - -## [v3.2.0] - 2023-08-19 - - -## [v3.2.0-rc.0] - 2023-08-17 - - -## v3.2.0-alpha.0 - 2023-08-16 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.2.2-alpha.0...HEAD -[v3.2.2-alpha.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0...v3.2.2-alpha.0 -[v3.2.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-rc.0...v3.2.0 -[v3.2.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.2.0-alpha.0...v3.2.0-rc.0 diff --git a/CHANGELOG/CHANGELOG-3.3.md b/CHANGELOG/CHANGELOG-3.3.md deleted file mode 100644 index f7326691ac..0000000000 --- a/CHANGELOG/CHANGELOG-3.3.md +++ /dev/null @@ -1,40 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.3.1] - 2023-09-13 - - -## [v3.3.1-beta.0] - 2023-09-11 - - -## [v3.3.0-rc.1] - 2023-09-11 - - -## [v3.3.0-rc.12] - 2023-09-11 - - -## [v3.3.0] - 2023-09-09 - - -## v3.3.0-rc.0 - 2023-09-07 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.3.1...HEAD -[v3.3.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.1-beta.0...v3.3.1 -[v3.3.1-beta.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.1...v3.3.1-beta.0 -[v3.3.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.12...v3.3.0-rc.1 -[v3.3.0-rc.12]: https://github.com/openimsdk/open-im-server/compare/v3.3.0...v3.3.0-rc.12 -[v3.3.0]: https://github.com/openimsdk/open-im-server/compare/v3.3.0-rc.0...v3.3.0 diff --git a/CHANGELOG/CHANGELOG-3.4.md b/CHANGELOG/CHANGELOG-3.4.md deleted file mode 100644 index db4dd59c53..0000000000 --- a/CHANGELOG/CHANGELOG-3.4.md +++ /dev/null @@ -1,32 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.4.2] - 2023-12-14 - - -## [v3.4.0] - 2023-11-10 - - -## [v3.4.0-rc.1] - 2023-11-09 - - -## v3.4.0-rc.0 - 2023-11-09 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.4.2...HEAD -[v3.4.2]: https://github.com/openimsdk/open-im-server/compare/v3.4.0...v3.4.2 -[v3.4.0]: https://github.com/openimsdk/open-im-server/compare/v3.4.0-rc.1...v3.4.0 -[v3.4.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.4.0-rc.0...v3.4.0-rc.1 diff --git a/CHANGELOG/CHANGELOG-3.5.md b/CHANGELOG/CHANGELOG-3.5.md deleted file mode 100644 index b929922f16..0000000000 --- a/CHANGELOG/CHANGELOG-3.5.md +++ /dev/null @@ -1,84 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## [v3.5.1-alpha.2] - 2024-01-26 - - -## [v3.5.1-rc.1] - 2024-01-23 - - -## [v3.5.1-alpha.1] - 2024-01-09 - - -## [v3.5.0] - 2024-01-02 - - -## [v3.5.1] - 2024-01-02 - - -## [v3.5.1-bate.1] - 2024-01-02 - - -## [v3.5.1-rc.0] - 2023-12-30 - - -## [v3.5.0-rc.8] - 2023-12-28 - - -## [v3.5.0-rc.7] - 2023-12-18 - - -## [v3.5.0-rc.6] - 2023-12-15 - - -## [v3.5.0-rc.5] - 2023-12-15 - - -## [v3.5.0-rc.4] - 2023-12-14 - - -## [v3.5.0-rc.3] - 2023-12-14 - - -## [v3.5.0-rc.2] - 2023-12-14 - - -## [v3.5.0-rc.1] - 2023-12-14 - - -## [v3.5.0-rc.0] - 2023-12-14 - - -## v3.5.0-beta.1 - 2023-11-29 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-alpha.2...HEAD -[v3.5.1-alpha.2]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-rc.1...v3.5.1-alpha.2 -[v3.5.1-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-alpha.1...v3.5.1-rc.1 -[v3.5.1-alpha.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.0...v3.5.1-alpha.1 -[v3.5.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.1...v3.5.0 -[v3.5.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-bate.1...v3.5.1 -[v3.5.1-bate.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.1-rc.0...v3.5.1-bate.1 -[v3.5.1-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.8...v3.5.1-rc.0 -[v3.5.0-rc.8]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.7...v3.5.0-rc.8 -[v3.5.0-rc.7]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.6...v3.5.0-rc.7 -[v3.5.0-rc.6]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.5...v3.5.0-rc.6 -[v3.5.0-rc.5]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.4...v3.5.0-rc.5 -[v3.5.0-rc.4]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.3...v3.5.0-rc.4 -[v3.5.0-rc.3]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.2...v3.5.0-rc.3 -[v3.5.0-rc.2]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.1...v3.5.0-rc.2 -[v3.5.0-rc.1]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-rc.0...v3.5.0-rc.1 -[v3.5.0-rc.0]: https://github.com/openimsdk/open-im-server/compare/v3.5.0-beta.1...v3.5.0-rc.0 diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md deleted file mode 100644 index 214d583401..0000000000 --- a/CHANGELOG/CHANGELOG-3.6.md +++ /dev/null @@ -1,20 +0,0 @@ -# Version logging for OpenIM - - - - - - -## [Unreleased] - - - -## v3.6.0 - 2024-03-07 -### Reverts -- update etcd to v3.5.2 ([#206](https://github.com/openimsdk/open-im-server/issues/206)) - -### Pull Requests -- Merge branch 'tuoyun' - - -[Unreleased]: https://github.com/openimsdk/open-im-server/compare/v3.6.0...HEAD diff --git a/CHANGELOG/CHANGELOG.md b/CHANGELOG/CHANGELOG.md deleted file mode 100644 index f5d62ab242..0000000000 --- a/CHANGELOG/CHANGELOG.md +++ /dev/null @@ -1,173 +0,0 @@ -# Changelog - -- [Changelog](#changelog) - - [OpenIM versioning policy](#openim-versioning-policy) - - [command](#command) - - [install](#install) - - [User](#user) - - [create next tag](#create-next-tag) - - [Release version logs](#release-version-logs) - - [Introduction](#introduction) - - [Naming Format](#naming-format) - - [Examples](#examples) - - [Version Modifiers](#version-modifiers) - - [Versioning Strategy](#versioning-strategy) - - -All notable changes to this project will be documented in this file. - -+ [https://github.com/openimsdk/open-im-server/releases](https://github.com/openimsdk/open-im-server/releases) - -## OpenIM versioning policy - -+ [OpenIM Version](../docs/contrib/version.md) - -## command - -To use git-chglog you need to configure: - -1. CHANGELOG templates -2. git-chglog configuration - -### install - -```bash -$ go get github.com/git-chglog/git-chglog/cmd/git-chglog -``` - - -## User - -```bash -$ git-chglog --init -``` - -**Options** - -- What is the URL of your repository?: https://github.com/openimsdk/open-im-server -- What is your favorite style?: github -- Choose the format of your favorite commit message: (): -- feat(core): Add new feature -- What is your favorite template style?: standard -- Do you include Merge Commit in CHANGELOG?: n -- Do you include Revert Commit in CHANGELOG?: y -- In which directory do you output configuration files and templates?: .chglog - -```bash -git-chglog --tag-filter-pattern 'v2.0.*' -o CHANGELOG-2.0.md -``` - -**Other uses:** - -```bash -$ git-chglog - - If is not specified, it corresponds to all tags. - This is the simplest example. - -$ git-chglog 1.0.0..2.0.0 - - The above is a command to generate CHANGELOG including commit of 1.0.0 to 2.0.0. - -$ git-chglog 1.0.0 - - The above is a command to generate CHANGELOG including commit of only 1.0.0. - -$ git-chglog $(git describe --tags $(git rev-list --tags --max-count=1)) - - The above is a command to generate CHANGELOG with the commit included in the latest tag. - -$ git-chglog --output CHANGELOG.md - - The above is a command to output to CHANGELOG.md instead of standard output. - -$ git-chglog --config custom/dir/config.yml - - The above is a command that uses a configuration file placed other than ".chglog/config.yml". -``` - - -## create next tag - -```bash -$ git-chglog --next-tag 2.0.0 -o CHANGELOG.md -$ git commit -am "release 2.0.0" -$ git tag 2.0.0 -``` - -| Query | Description | Example | -| -------------- | ---------------------------------------------- | --------------------------- | -| `..` | Commit contained in `` tags from ``. | `$ git-chglog 1.0.0..2.0.0` | -| `..` | Commit from the `` to the latest tag. | `$ git-chglog 1.0.0..` | -| `..` | Commit from the oldest tag to ``. | `$ git-chglog ..2.0.0` | -| `` | Commit contained in ``. | `$ git-chglog 1.0.0` | - - -## Release version logs - -+ [OpenIM CHANGELOG-V1.0](CHANGELOG-1.0.md) -+ [OpenIM CHANGELOG-V2.0](CHANGELOG-2.0.md) -+ [OpenIM CHANGELOG-V2.1](CHANGELOG-2.1.md) -+ [OpenIM CHANGELOG-V2.2](CHANGELOG-2.2.md) -+ [OpenIM CHANGELOG-V2.3](CHANGELOG-2.3.md) -+ [OpenIM CHANGELOG-V2.9](CHANGELOG-2.9.md) -+ [OpenIM CHANGELOG-V3.0](CHANGELOG-3.0.md) -+ [OpenIM CHANGELOG-V3.1](CHANGELOG-3.1.md) -+ [OpenIM CHANGELOG-V3.2](CHANGELOG-3.2.md) -+ [OpenIM CHANGELOG-V3.3](CHANGELOG-3.3.md) - - -## Introduction - -In both the open-source and closed-source software development communities, it is important to follow a consistent and understandable versioning scheme for software projects. This ensures clear communication of changes, compatibility, and stability across different releases. One widely adopted naming convention is the Semantic Versioning 2.0.0. - -## Naming Format - -The most common format for version numbers is as follows: - -```bash -major.minor[.patch[.build]] -``` - -Let's take a closer look at each component: - -1. **Major Version**: This is the first number in the versioning scheme and indicates significant changes that may not be backward compatible (specific to each project). -2. **Minor Version**: The second number signifies the addition of new features while maintaining backward compatibility. -3. **Patch Version**: The third number represents bug fixes or code optimizations without introducing new features. It is generally backward compatible. -4. **Build Version**: Typically an automatically generated number that increments with each code commit. - -## Examples - -Here are a few examples to illustrate the versioning scheme: - -1. `1.0` -2. `2.14.0.1478` -3. `3.2.1 build-354` - -## Version Modifiers - -Apart from the version numbers, there are also version modifiers used to indicate specific stages or statuses of a release. Some commonly used version modifiers include: - -- **alpha**: An internal testing version with numerous known bugs. It is primarily used for communication among developers. -- **beta**: A testing version released to enthusiastic users for feedback and bug detection. -- **rc (release candidate)**: The final testing version before the official release. -- **ga (general availability)**: The initial stable release for public distribution. -- **r/release** (or no modifier at all): The final released version intended for general users. -- **lts (long-term support)**: Designates a version that will receive extended maintenance and bug fixes for a specified number of years. - -## Versioning Strategy - -To effectively manage version numbers, the following strategies are commonly employed: - -- The initial version of a project can be either `0.1` or `1.0`. -- When fixing bugs, the patch version is incremented by 1. -- When adding new features, the minor version is incremented by 1, and the patch version is reset to 0. -- In the case of significant modifications, the major version is incremented by 1. -- The build version is usually automatically generated by the compilation process and follows a defined format. It does not require manual control. - -By adhering to these strategies and guidelines, developers can maintain consistency and clarity in versioning their software projects. This enables users and collaborators to understand the nature of changes between different releases and ensure compatibility with their systems. - -(Note: Markdown formatting has been used to structure this article. Markdown is a lightweight markup language used to format text on platforms like GitHub.) - ------- - -**Note**: The above article is based on the given content and aims to provide a Markdown-formatted English article explaining the naming conventions for software project versions, specifically focusing on the Semantic Versioning 2.0.0. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3f765805c3..e082dd64c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.13 +RUN go get github.com/openimsdk/gomake@v0.0.14-alpha.5 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/README.md b/README.md index d73d5749a5..a99559cdb0 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/README_zh_CN.md b/README_zh_CN.md index 65aac9ebc6..59198eafb6 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index 58e540c05f..e29ed2a592 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -25,5 +25,4 @@ func main() { if err := cmd.NewApiCmd().Exec(); err != nil { program.ExitWithError(err) } - } diff --git a/config/alertmanager.yml b/config/alertmanager.yml new file mode 100644 index 0000000000..6c675ab6f1 --- /dev/null +++ b/config/alertmanager.yml @@ -0,0 +1,34 @@ +global: + resolve_timeout: 5m + smtp_from: alert@openim.io + smtp_smarthost: smtp.163.com:465 + smtp_auth_username: alert@openim.io + smtp_auth_password: YOURAUTHPASSWORD + smtp_require_tls: false + smtp_hello: xxx + +templates: + - /etc/alertmanager/email.tmpl + +route: + group_by: [ 'alertname' ] + group_wait: 5s + group_interval: 5s + repeat_interval: 5m + receiver: email + routes: + - matchers: + - alertname = "XXX" + group_by: [ 'instance' ] + group_wait: 5s + group_interval: 5s + repeat_interval: 5m + receiver: email + +receivers: + - name: email + email_configs: + - to: 'alert@example.com' + html: '{{ template "email.to.html" . }}' + headers: { Subject: "[OPENIM-SERVER]Alarm" } + send_resolved: true diff --git a/config/discovery.yml b/config/discovery.yml index 3d96ff9b66..78a36f3d1f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,4 +1,4 @@ -enable: "etcd" +enable: etcd etcd: rootDirectory: openim address: [ localhost:12379 ] diff --git a/config/email.tmpl b/config/email.tmpl new file mode 100644 index 0000000000..824144e9d7 --- /dev/null +++ b/config/email.tmpl @@ -0,0 +1,36 @@ +{{ define "email.to.html" }} +{{ if eq .Status "firing" }} + {{ range .Alerts }} + +

+

OpenIM Alert

+

Alert Status: firing

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ {{ end }} + + +{{ else if eq .Status "resolved" }} + {{ range .Alerts }} + +
+

OpenIM Alert

+

Alert Status: resolved

+

Alert Program: Prometheus Alert

+

Severity Level: {{ .Labels.severity }}

+

Alert Type: {{ .Labels.alertname }}

+

Affected Host: {{ .Labels.instance }}

+

Affected Service: {{ .Labels.job }}

+

Alert Subject: {{ .Annotations.summary }}

+

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

+
+ {{ end }} + +{{ end }} +{{ end }} diff --git a/config/grafana-template/Demo.json b/config/grafana-template/Demo.json new file mode 100644 index 0000000000..ea17d2c0ab --- /dev/null +++ b/config/grafana-template/Demo.json @@ -0,0 +1,5576 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Is the service up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 9, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "up", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "UP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of online users and login users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "online users" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#37bbff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "online_user_num", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "online users", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(user_login_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "login num", + "range": true, + "refId": "B" + } + ], + "title": "Login Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of register users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "register users" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7437ff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "user_register_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "register users", + "range": true, + "refId": "A" + } + ], + "title": "Register num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg success.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Success Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg failed .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "single msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#ff00dc", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "group msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0cffef", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of msg failed offline pushed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_offline_push_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Msg Offline Push Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed set seq.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(seq_set_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Seq Set Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of messages that take a long time to send.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "msg_long_time_push_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Long Time Send Msg Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of successfully inserted messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Success Insert Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed insertion messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Failed Insert Num", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 22, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of call of all API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (api_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Requests Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of call of all API within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "/friend/get_friend_list" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(api_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Requests Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of err return of API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (api_count{code != \"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Error Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of err return of API with err code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (api_count{code != \"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "API Error Total With Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the qps of API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1ed9d4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(api_count[1m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "qps", + "range": true, + "refId": "A" + } + ], + "title": "API QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of err return of API within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(api_count{code != \"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Error Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of err return of API with err code within the time frame..", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (increase(api_count{code != \"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "API Error Num With Code", + "type": "timeseries" + } + ], + "title": "API", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of call of all RPC.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (rpc_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Total Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the error return of RPC.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (rpc_count{code!=\"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the error return of RPC with code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (rpc_count{code!=\"0\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Count With Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of call of all RPC within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Total Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of RPC calls within the time frame, aggregated by name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (name) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Num by Name", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of call of RPC within the time frame, aggregated by address.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance) (increase(rpc_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Num by Address", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the error return of RPC within the time frame within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path) (increase(rpc_count{code!=\"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the error return of RPC with code within the time frame within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (path, code) (increase(rpc_count{code!=\"0\"}[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{path}}: code={{code}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Error Num With Code", + "type": "timeseries" + } + ], + "title": "RPC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 25, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of HTTP requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path) (http_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of HTTP requests with status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path, status) (http_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}: {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Count With Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of HTTP requests within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path) (increase(http_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of HTTP requests with status within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, path, status) (increase(http_count[$time]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{method}}: {{path}}: {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Total Num With Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the qps of HTTP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1ed9d4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(http_count[1m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "qps", + "range": true, + "refId": "A" + } + ], + "title": "HTTP QPS", + "type": "timeseries" + } + ], + "title": "HTTP", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 6, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 3, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(go_gc_duration_seconds_count{job=~\"$rpcNameFilter\"}[5m]),\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "GC Rate Per Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Measures the frequency of garbage collection operations in the Go environment, averaged over the last five minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "label_replace(\r\n rate(go_gc_duration_seconds_count{job!~\"$rpcNameFilter\"}[5m]),\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "hide": false, + "instant": false, + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "GC Rate Per Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of goroutines.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_goroutines{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of goroutines.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_goroutines{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes allocated and still in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_alloc_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Alloc Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes allocated and still in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_alloc_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Alloc Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes used by the profiling bucket hash table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_buck_hash_sys_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Buck Hash Sys Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes used by the profiling bucket hash table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_buck_hash_sys_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Buck Hash Sys Bytes ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes in use by mcache structures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_mcache_inuse_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Mcache Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of bytes in use by mcache structures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n go_memstats_mcache_inuse_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "Go Mcache Bytes", + "type": "timeseries" + } + ], + "title": "GO infomation", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "openimserver-openim-rpc.*", + "value": "openimserver-openim-rpc.*" + }, + "hide": 0, + "includeAll": false, + "label": "filter", + "multi": false, + "name": "rpcNameFilter", + "options": [ + { + "selected": true, + "text": "openimserver-openim-rpc.*", + "value": "openimserver-openim-rpc.*" + } + ], + "query": "openimserver-openim-rpc.*", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "{{job}}: {{instance}}", + "value": "{{job}}: {{instance}}" + }, + "description": "common legend name", + "hide": 0, + "includeAll": false, + "label": "legend", + "multi": false, + "name": "legendName", + "options": [ + { + "selected": true, + "text": "{{job}}: {{instance}}", + "value": "{{job}}: {{instance}}" + } + ], + "query": "{{job}}: {{instance}}", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "description": "Global promQL time range.", + "hide": 0, + "includeAll": false, + "label": "time", + "multi": false, + "name": "time", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "24h", + "value": "24h" + }, + { + "selected": false, + "text": "1w", + "value": "1w" + }, + { + "selected": false, + "text": "4w", + "value": "4w" + }, + { + "selected": false, + "text": "12w", + "value": "12w" + }, + { + "selected": false, + "text": "24w", + "value": "24w" + }, + { + "selected": false, + "text": "1y", + "value": "1y" + }, + { + "selected": false, + "text": "2y", + "value": "2y" + }, + { + "selected": false, + "text": "4y", + "value": "4y" + }, + { + "selected": false, + "text": "10y", + "value": "10y" + } + ], + "query": "1m,5m,30m,1h,3h,6h,12h,24h,1w,4w,12w,24w,1y,2y,4y,10y", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Demo", + "uid": "a506d250-b606-4702-86a7-ac6aa1d069a1", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/config/instance-down-rules.yml b/config/instance-down-rules.yml new file mode 100644 index 0000000000..bcac7ba60d --- /dev/null +++ b/config/instance-down-rules.yml @@ -0,0 +1,44 @@ +groups: + - name: instance_down + rules: + - alert: InstanceDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes." + + - name: database_insert_failure_alerts + rules: + - alert: DatabaseInsertFailed + expr: (increase(msg_insert_redis_failed_total[5m]) > 0) or (increase(msg_insert_mongo_failed_total[5m]) > 0) + for: 1m + labels: + severity: critical + annotations: + summary: "Increase in MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter detected" + description: "Either MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter has increased in the last 5 minutes, indicating failures in message insert operations to Redis or MongoDB,maybe the redis or mongodb is crash." + + - name: registrations_few + rules: + - alert: RegistrationsFew + expr: increase(user_login_total[1h]) == 0 + for: 1m + labels: + severity: info + annotations: + summary: "Too few registrations within the time frame" + description: "The number of registrations in the last hour is 0. There might be some issues." + + - name: messages_few + rules: + - alert: MessagesFew + expr: (increase(single_chat_msg_process_success_total[1h])+increase(group_chat_msg_process_success_total[1h])) == 0 + for: 1m + labels: + severity: info + annotations: + summary: "Too few messages within the time frame" + description: "The number of messages sent in the last hour is 0. There might be some issues." diff --git a/config/kafka.yml b/config/kafka.yml index d412e1be06..fd06ae2bb4 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -3,34 +3,38 @@ username: '' # Password for authentication password: '' # Producer acknowledgment settings -producerAck: "" +producerAck: # Compression type to use (e.g., none, gzip, snappy) -compressType: "none" +compressType: none # List of Kafka broker addresses address: [ localhost:19094 ] # Kafka topic for Redis integration -toRedisTopic: "toRedis" +toRedisTopic: toRedis # Kafka topic for MongoDB integration -toMongoTopic: "toMongo" +toMongoTopic: toMongo # Kafka topic for push notifications -toPushTopic: "toPush" +toPushTopic: toPush +# Kafka topic for offline push notifications +toOfflinePushTopic: toOfflinePush # Consumer group ID for Redis topic toRedisGroupID: redis # Consumer group ID for MongoDB topic toMongoGroupID: mongo # Consumer group ID for push notifications topic toPushGroupID: push +# Consumer group ID for offline push notifications topic +toOfflinePushGroupID: offlinePush # TLS (Transport Layer Security) configuration tls: # Enable or disable TLS enableTLS: false # CA certificate file path - caCrt: "" + caCrt: # Client certificate file path - clientCrt: "" + clientCrt: # Client key file path - clientKey: "" + clientKey: # Client key password - clientKeyPwd: "" + clientKeyPwd: # Whether to skip TLS verification (not recommended for production) insecureSkipVerify: false diff --git a/config/log.yml b/config/log.yml index 2194d89174..8620af611b 100644 --- a/config/log.yml +++ b/config/log.yml @@ -10,4 +10,5 @@ remainLogLevel: 6 isStdout: false # Whether to log in JSON format, default is acceptable isJson: false - +# output simplify log when KeyAndValues's value len is bigger than 50 in rpc method log +isSimplify: true \ No newline at end of file diff --git a/config/minio.yml b/config/minio.yml index 11a9ace354..ad1a32a8c2 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -1,15 +1,15 @@ # Name of the bucket in MinIO -bucket: "openim" +bucket: openim # Access key ID for MinIO authentication -accessKeyID: "root" +accessKeyID: root # Secret access key for MinIO authentication -secretAccessKey: "openIM123" +secretAccessKey: openIM123 # Session token for MinIO authentication (optional) -sessionToken: '' +sessionToken: # Internal address of the MinIO server -internalAddress: "localhost:10005" +internalAddress: localhost:10005 # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://external_ip:10005" +externalAddress: http://external_ip:10005 # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 98f5694e45..78f85992c9 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,5 +1,5 @@ # URI for database connection, leave empty if using address and credential settings directly -uri: '' +uri: # List of MongoDB server addresses address: [ localhost:37017 ] # Name of the database diff --git a/config/notification.yml b/config/notification.yml index 278376c244..85ca91af18 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -28,11 +28,11 @@ groupCreated: # Enables or disables offline push notifications. enable: false # Title for the notification when a group is created. - title: "create group title" + title: create group title # Description for the notification. - desc: "create group desc" + desc: create group desc # Additional information for the notification. - ext: "create group ext" + ext: create group ext groupInfoSet: isSendMsg: false @@ -40,9 +40,9 @@ groupInfoSet: unreadCount: false offlinePush: enable: false - title: "groupInfoSet title" - desc: "groupInfoSet desc" - ext: "groupInfoSet ext" + title: groupInfoSet title + desc: groupInfoSet desc + ext: groupInfoSet ext joinGroupApplication: @@ -51,9 +51,9 @@ joinGroupApplication: unreadCount: false offlinePush: enable: false - title: "joinGroupApplication title" - desc: "joinGroupApplication desc" - ext: "joinGroupApplication ext" + title: joinGroupApplication title + desc: joinGroupApplication desc + ext: joinGroupApplication ext memberQuit: isSendMsg: true @@ -61,9 +61,9 @@ memberQuit: unreadCount: false offlinePush: enable: false - title: "memberQuit title" - desc: "memberQuit desc" - ext: "memberQuit ext" + title: memberQuit title + desc: memberQuit desc + ext: memberQuit ext groupApplicationAccepted: isSendMsg: false @@ -71,9 +71,9 @@ groupApplicationAccepted: unreadCount: false offlinePush: enable: false - title: "groupApplicationAccepted title" - desc: "groupApplicationAccepted desc" - ext: "groupApplicationAccepted ext" + title: groupApplicationAccepted title + desc: groupApplicationAccepted desc + ext: groupApplicationAccepted ext groupApplicationRejected: isSendMsg: false @@ -81,9 +81,9 @@ groupApplicationRejected: unreadCount: false offlinePush: enable: false - title: "groupApplicationRejected title" - desc: "groupApplicationRejected desc" - ext: "groupApplicationRejected ext" + title: groupApplicationRejected title + desc: groupApplicationRejected desc + ext: groupApplicationRejected ext groupOwnerTransferred: @@ -92,9 +92,9 @@ groupOwnerTransferred: unreadCount: false offlinePush: enable: false - title: "groupOwnerTransferred title" - desc: "groupOwnerTransferred desc" - ext: "groupOwnerTransferred ext" + title: groupOwnerTransferred title + desc: groupOwnerTransferred desc + ext: groupOwnerTransferred ext memberKicked: isSendMsg: true @@ -102,9 +102,9 @@ memberKicked: unreadCount: false offlinePush: enable: false - title: "memberKicked title" - desc: "memberKicked desc" - ext: "memberKicked ext" + title: memberKicked title + desc: memberKicked desc + ext: memberKicked ext memberInvited: isSendMsg: true @@ -112,9 +112,9 @@ memberInvited: unreadCount: false offlinePush: enable: false - title: "memberInvited title" - desc: "memberInvited desc" - ext: "memberInvited ext" + title: memberInvited title + desc: memberInvited desc + ext: memberInvited ext memberEnter: isSendMsg: true @@ -122,9 +122,9 @@ memberEnter: unreadCount: false offlinePush: enable: false - title: "memberEnter title" - desc: "memberEnter desc" - ext: "memberEnter ext" + title: memberEnter title + desc: memberEnter desc + ext: memberEnter ext groupDismissed: isSendMsg: true @@ -132,9 +132,9 @@ groupDismissed: unreadCount: false offlinePush: enable: false - title: "groupDismissed title" - desc: "groupDismissed desc" - ext: "groupDismissed ext" + title: groupDismissed title + desc: groupDismissed desc + ext: groupDismissed ext groupMuted: isSendMsg: true @@ -142,9 +142,9 @@ groupMuted: unreadCount: false offlinePush: enable: false - title: "groupMuted title" - desc: "groupMuted desc" - ext: "groupMuted ext" + title: groupMuted title + desc: groupMuted desc + ext: groupMuted ext groupCancelMuted: isSendMsg: true @@ -152,11 +152,11 @@ groupCancelMuted: unreadCount: false offlinePush: enable: false - title: "groupCancelMuted title" - desc: "groupCancelMuted desc" - ext: "groupCancelMuted ext" + title: groupCancelMuted title + desc: groupCancelMuted desc + ext: groupCancelMuted ext defaultTips: - tips: "group Cancel Muted" + tips: group Cancel Muted groupMemberMuted: @@ -165,9 +165,9 @@ groupMemberMuted: unreadCount: false offlinePush: enable: false - title: "groupMemberMuted title" - desc: "groupMemberMuted desc" - ext: "groupMemberMuted ext" + title: groupMemberMuted title + desc: groupMemberMuted desc + ext: groupMemberMuted ext groupMemberCancelMuted: isSendMsg: true @@ -175,9 +175,9 @@ groupMemberCancelMuted: unreadCount: false offlinePush: enable: false - title: "groupMemberCancelMuted title" - desc: "groupMemberCancelMuted desc" - ext: "groupMemberCancelMuted ext" + title: groupMemberCancelMuted title + desc: groupMemberCancelMuted desc + ext: groupMemberCancelMuted ext groupMemberInfoSet: isSendMsg: false @@ -185,9 +185,9 @@ groupMemberInfoSet: unreadCount: false offlinePush: enable: false - title: "groupMemberInfoSet title" - desc: "groupMemberInfoSet desc" - ext: "groupMemberInfoSet ext" + title: groupMemberInfoSet title + desc: groupMemberInfoSet desc + ext: groupMemberInfoSet ext groupInfoSetAnnouncement: isSendMsg: true @@ -195,9 +195,9 @@ groupInfoSetAnnouncement: unreadCount: false offlinePush: enable: false - title: "groupInfoSetAnnouncement title" - desc: "groupInfoSetAnnouncement desc" - ext: "groupInfoSetAnnouncement ext" + title: groupInfoSetAnnouncement title + desc: groupInfoSetAnnouncement desc + ext: groupInfoSetAnnouncement ext groupInfoSetName: @@ -206,9 +206,9 @@ groupInfoSetName: unreadCount: false offlinePush: enable: false - title: "groupInfoSetName title" - desc: "groupInfoSetName desc" - ext: "groupInfoSetName ext" + title: groupInfoSetName title + desc: groupInfoSetName desc + ext: groupInfoSetName ext #############################friend################################# @@ -218,9 +218,9 @@ friendApplicationAdded: unreadCount: false offlinePush: enable: false - title: "Somebody applies to add you as a friend" - desc: "Somebody applies to add you as a friend" - ext: "Somebody applies to add you as a friend" + title: Somebody applies to add you as a friend + desc: Somebody applies to add you as a friend + ext: Somebody applies to add you as a friend friendApplicationApproved: isSendMsg: true @@ -228,9 +228,9 @@ friendApplicationApproved: unreadCount: false offlinePush: enable: true - title: "Someone applies to add your friend application" - desc: "Someone applies to add your friend application" - ext: "Someone applies to add your friend application" + title: Someone applies to add your friend application + desc: Someone applies to add your friend application + ext: Someone applies to add your friend application friendApplicationRejected: isSendMsg: false @@ -238,9 +238,9 @@ friendApplicationRejected: unreadCount: false offlinePush: enable: true - title: "Someone rejected your friend application" - desc: "Someone rejected your friend application" - ext: "Someone rejected your friend application" + title: Someone rejected your friend application + desc: Someone rejected your friend application + ext: Someone rejected your friend application friendAdded: isSendMsg: false @@ -248,9 +248,9 @@ friendAdded: unreadCount: false offlinePush: enable: true - title: "We have become friends" - desc: "We have become friends" - ext: "We have become friends" + title: We have become friends + desc: We have become friends + ext: We have become friends friendDeleted: isSendMsg: false @@ -258,9 +258,9 @@ friendDeleted: unreadCount: false offlinePush: enable: true - title: "deleted a friend" - desc: "deleted a friend" - ext: "deleted a friend" + title: deleted a friend + desc: deleted a friend + ext: deleted a friend friendRemarkSet: isSendMsg: false @@ -268,9 +268,9 @@ friendRemarkSet: unreadCount: false offlinePush: enable: true - title: "Your friend's profile has been changed" - desc: "Your friend's profile has been changed" - ext: "Your friend's profile has been changed" + title: Your friend's profile has been changed + desc: Your friend's profile has been changed + ext: Your friend's profile has been changed blackAdded: isSendMsg: false @@ -278,9 +278,9 @@ blackAdded: unreadCount: false offlinePush: enable: true - title: "blocked a user" - desc: "blocked a user" - ext: "blocked a user" + title: blocked a user + desc: blocked a user + ext: blocked a user blackDeleted: isSendMsg: false @@ -288,9 +288,9 @@ blackDeleted: unreadCount: false offlinePush: enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" + title: Remove a blocked user + desc: Remove a blocked user + ext: Remove a blocked user friendInfoUpdated: isSendMsg: false @@ -298,9 +298,9 @@ friendInfoUpdated: unreadCount: false offlinePush: enable: true - title: "friend info updated" - desc: "friend info updated" - ext: "friend info updated" + title: friend info updated + desc: friend info updated + ext: friend info updated #####################user######################### userInfoUpdated: @@ -309,9 +309,9 @@ userInfoUpdated: unreadCount: false offlinePush: enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" + title: Remove a blocked user + desc: Remove a blocked user + ext: Remove a blocked user userStatusChanged: isSendMsg: false @@ -319,9 +319,9 @@ userStatusChanged: unreadCount: false offlinePush: enable: false - title: "user status changed" - desc: "user status changed" - ext: "user status changed" + title: user status changed + desc: user status changed + ext: user status changed #####################conversation######################### conversationChanged: @@ -330,9 +330,9 @@ conversationChanged: unreadCount: false offlinePush: enable: true - title: "conversation changed" - desc: "conversation changed" - ext: "conversation changed" + title: conversation changed + desc: conversation changed + ext: conversation changed conversationSetPrivate: isSendMsg: true @@ -340,6 +340,6 @@ conversationSetPrivate: unreadCount: false offlinePush: enable: true - title: "burn after reading" - desc: "burn after reading" - ext: "burn after reading" + title: burn after reading + desc: burn after reading + ext: burn after reading diff --git a/config/openim-api.yml b/config/openim-api.yml index 78a688fcd6..4c38e1005b 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -3,11 +3,14 @@ api: listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, must be consistent with the number of prometheus.ports ports: [ 10002 ] + # API compression level; 0: default compression, 1: best compression, 2: best speed, -1: no compression + compressionLevel: 0 + prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must match the number of api.ports - ports: [ 20113 ] + ports: [ 12002 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 9bbccfd252..c05bd2485f 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,2 +1,3 @@ -chatRecordsClearTime: "0 2 * * *" +cronExecuteTime: 0 2 * * * retainChatRecords: 365 +fileExpireTime: 90 diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 0c92d83278..428f3ba476 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,14 +1,14 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10140 ] + ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20112 ] + ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 @@ -25,6 +25,3 @@ longConnSvr: # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLoginPolicy: 1 - - - diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 07a7dc1ab1..753ac10bc0 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -3,4 +3,4 @@ prometheus: enable: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # Because four instances have been launched, four ports need to be specified - ports: [ 20108, 20109, 20110, 20111 ] + ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 9384008a04..6df2a62b7f 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -1,46 +1,41 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170 ] + ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20107 ] + ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185 ] maxConcurrentWorkers: 3 -#"Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified." -enable: "geTui" +#Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. +enable: geTui geTui: - pushUrl: "https://restapi.getui.com/v2/$appId" - masterSecret: '' - appKey: '' - intent: '' - channelID: '' - channelName: '' + pushUrl: https://restapi.getui.com/v2/$appId + masterSecret: + appKey: + intent: + channelID: + channelName: fcm: # Prioritize using file paths. If the file path is empty, use URL - filePath: "" # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. - authURL: "" # Must start with https or http. + filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. + authURL: # Must start with https or http. jpns: - appKey: '' - masterSecret: '' - pushURL: '' - pushIntent: '' + appKey: + masterSecret: + pushURL: + pushIntent: # iOS system push sound and badge count iosPush: - pushSound: "xxx" + pushSound: xxx badgeCount: true production: false - - - - - - +fullUserCache: true diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 2d861cd5ab..496803e43b 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -1,18 +1,17 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10160 ] + ports: [ 10200 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20106 ] + ports: [ 12200 ] tokenPolicy: # Token validity period, in days expire: 90 - diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index a094bfac10..3581d7e19e 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -1,13 +1,13 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10180 ] + ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20105 ] + ports: [ 12220 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 7b829f971c..3022c09f32 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -1,13 +1,13 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10120 ] + ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20104 ] + ports: [ 12240 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 78b44030e0..9a634d12ff 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -1,13 +1,16 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10150 ] + ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20103 ] + ports: [ 12260 ] + + +enableHistoryForNewMembers: true diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 17ce26e9b2..82d6e2f539 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -1,20 +1,17 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10130 ] + ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20102 ] + ports: [ 12280 ] # Does sending messages require friend verification friendVerify: false - - - diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index bde38ccc44..d8f2d427f2 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -1,32 +1,40 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: '' + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10190 ] + ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20101 ] + ports: [ 12300 ] object: # Use MinIO as object storage, or set to "cos", "oss", "kodo", "aws", while also configuring the corresponding settings - enable: "minio" + enable: minio cos: bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com - secretID: '' - secretKey: '' - sessionToken: '' + secretID: + secretKey: + sessionToken: publicRead: false oss: - endpoint: "https://oss-cn-chengdu.aliyuncs.com" - bucket: "demo-9999999" - bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" - accessKeyID: '' - accessKeySecret: '' - sessionToken: '' - publicRead: false \ No newline at end of file + endpoint: https://oss-cn-chengdu.aliyuncs.com + bucket: demo-9999999 + bucketURL: https://demo-9999999.oss-cn-chengdu.aliyuncs.com + accessKeyID: + accessKeySecret: + sessionToken: + publicRead: false + kodo: + endpoint: http://s3.cn-south-1.qiniucs.com + bucket: kodo-bucket-test + bucketURL: http://kodo-bucket-test-oetobfb.qiniudns.com + accessKeyID: + accessKeySecret: + sessionToken: + publicRead: false diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index cbfb55b6c7..798105472c 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -1,17 +1,13 @@ rpc: # API or other RPCs can access this RPC through this IP; if left blank, the internal network IP is obtained by default - registerIP: '' + registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports - ports: [ 10110 ] + ports: [ 10320 ] prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports - ports: [ 20100 ] - - - - + ports: [ 12320 ] diff --git a/config/prometheus.yml b/config/prometheus.yml new file mode 100644 index 0000000000..4f0f7e32c0 --- /dev/null +++ b/config/prometheus.yml @@ -0,0 +1,86 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: [internal_ip:19093] + +# Load rules once and periodically evaluate them according to the global evaluation_interval. +rule_files: + - instance-down-rules.yml +# - first_rules.yml +# - second_rules.yml + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label "job=job_name" to any timeseries scraped from this config. + # Monitored information captured by prometheus + + # prometheus fetches application services + - job_name: node_exporter + static_configs: + - targets: [ internal_ip:20500 ] + - job_name: openimserver-openim-api + static_configs: + - targets: [ internal_ip:12002 ] + labels: + namespace: default + - job_name: openimserver-openim-msggateway + static_configs: + - targets: [ internal_ip:12140 ] +# - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] + labels: + namespace: default + - job_name: openimserver-openim-msgtransfer + static_configs: + - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] +# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] + labels: + namespace: default + - job_name: openimserver-openim-push + static_configs: + - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12181, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-auth + static_configs: + - targets: [ internal_ip:12200 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-conversation + static_configs: + - targets: [ internal_ip:12220 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-friend + static_configs: + - targets: [ internal_ip:12240 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-group + static_configs: + - targets: [ internal_ip:12260 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-msg + static_configs: + - targets: [ internal_ip:12280 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-third + static_configs: + - targets: [ internal_ip:12300 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-user + static_configs: + - targets: [ internal_ip:12320 ] + labels: + namespace: default \ No newline at end of file diff --git a/config/redis.yml b/config/redis.yml index 6fe0dd02d4..2448bcb5c6 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,6 +1,7 @@ address: [ localhost:16379 ] -username: '' +username: password: openIM123 clusterMode: false db: 0 -maxRetry: 10 \ No newline at end of file +maxRetry: 10 +poolSize: 100 diff --git a/config/share.yml b/config/share.yml index fc97b6a1ff..4c5892615b 100644 --- a/config/share.yml +++ b/config/share.yml @@ -10,5 +10,5 @@ rpcRegisterName: conversation: conversation third: third -imAdminUserID: [ "imAdmin" ] +imAdminUserID: [ imAdmin ] diff --git a/config/webhooks.yml b/config/webhooks.yml index c7839d4f21..24fb2413a0 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -1,4 +1,4 @@ -url: "webhook://127.0.0.1:10008/callbackExample" +url: webhook://127.0.0.1:10008/callbackExample beforeSendSingleMsg: enable: false timeout: 5 @@ -13,6 +13,9 @@ afterUpdateUserInfoEx: afterSendSingleMsg: enable: false timeout: 5 + # Only the senID/recvID specified in attentionIds will send the callback + # if not set, all user messages will be callback + attentionIds: [] beforeSendGroupMsg: enable: false timeout: 5 @@ -127,6 +130,13 @@ beforeSetGroupInfo: enable: false timeout: 5 failedContinue: true +afterSetGroupInfoEX: + enable: false + timeout: 5 +beforeSetGroupInfoEX: + enable: false + timeout: 5 + failedContinue: true afterRevokeMsg: enable: false timeout: 5 diff --git a/deployments/templates/env-template.yaml b/deployments/templates/env-template.yaml index b1fb8b78be..b019f97e5b 100644 --- a/deployments/templates/env-template.yaml +++ b/deployments/templates/env-template.yaml @@ -129,7 +129,7 @@ REDIS_PORT=${REDIS_PORT} # Default: REDIS_PASSWORD=openIM123 REDIS_PASSWORD=${REDIS_PASSWORD} -# Kakfa username to authenticate with the Kafka service. +# Kafka username to authenticate with the Kafka service. # KAFKA_USERNAME=${KAFKA_USERNAME} # Port on which Kafka distributed streaming platform is running. diff --git a/docker-compose.yml b/docker-compose.yml index d72c1a2fa4..512f951dbd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,5 +140,49 @@ services: networks: - openim - +# prometheus: +# image: ${PROMETHEUS_IMAGE} +# container_name: prometheus +# restart: always +# volumes: +# - ./config/prometheus.yml:/etc/prometheus/prometheus.yml +# - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml +# - ${DATA_DIR}/components/prometheus/data:/prometheus +# command: +# - '--config.file=/etc/prometheus/prometheus.yml' +# - '--storage.tsdb.path=/prometheus' +# ports: +# - "19091:9090" +# networks: +# - openim +# +# alertmanager: +# image: ${ALERTMANAGER_IMAGE} +# container_name: alertmanager +# restart: always +# volumes: +# - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml +# - ./config/email.tmpl:/etc/alertmanager/email.tmpl +# ports: +# - "19093:9093" +# networks: +# - openim +# +# grafana: +# image: ${GRAFANA_IMAGE} +# container_name: grafana +# user: root +# restart: always +# environment: +# - GF_SECURITY_ALLOW_EMBEDDING=true +# - GF_SESSION_COOKIE_SAMESITE=none +# - GF_SESSION_COOKIE_SECURE=true +# - GF_AUTH_ANONYMOUS_ENABLED=true +# - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin +# ports: +# - "13000:3000" +# volumes: +# - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana +# networks: +# - openim diff --git a/docs/contrib/util-scripts.md b/docs/contrib/util-scripts.md index 0bf6f23e5c..30da871a49 100644 --- a/docs/contrib/util-scripts.md +++ b/docs/contrib/util-scripts.md @@ -32,7 +32,7 @@ This script offers a variety of utilities and helpers to enhance and simplify op ## brief descriptions of each function -**Englist:** +**English:** 1. `openim::util::ensure-gnu-sed` - Determines if GNU version of `sed` exists on the system and sets its name. 2. `openim::util::ensure-gnu-date` - Determines if GNU version of `date` exists on the system and sets its name. 3. `openim::util::check-file-in-alphabetical-order` - Checks if a file is sorted in alphabetical order. diff --git a/docs/contributing/CONTRIBUTING-JP.md b/docs/contributing/CONTRIBUTING-JP.md index 86bbfefcd7..1798d4e3d4 100644 --- a/docs/contributing/CONTRIBUTING-JP.md +++ b/docs/contributing/CONTRIBUTING-JP.md @@ -1,7 +1,7 @@ # How do I contribute code to OpenIM

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/contributing/CONTRIBUTING-PL.md b/docs/contributing/CONTRIBUTING-PL.md index 86bbfefcd7..1798d4e3d4 100644 --- a/docs/contributing/CONTRIBUTING-PL.md +++ b/docs/contributing/CONTRIBUTING-PL.md @@ -1,7 +1,7 @@ # How do I contribute code to OpenIM

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_cs.md b/docs/readme/README_cs.md index 5a9eeb2324..63f730a516 100644 --- a/docs/readme/README_cs.md +++ b/docs/readme/README_cs.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_da.md b/docs/readme/README_da.md index 1b776ddb8b..60d97348a4 100644 --- a/docs/readme/README_da.md +++ b/docs/readme/README_da.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index 252521f351..da01fcb470 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index cd1b7290ee..f123b85c36 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index 49f05cd4c3..7a1512e844 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index e707fc59b7..aaf7a9bd41 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_hu.md b/docs/readme/README_hu.md index 57f0066928..61013c3343 100644 --- a/docs/readme/README_hu.md +++ b/docs/readme/README_hu.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index bd94b11538..5a083c1bf8 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index bd7a1aed37..ebcdd71eee 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index ca2a816db0..3cf19f537e 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 30bc767305..81820590be 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index e500da6d2d..a6ab39253b 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -19,7 +19,7 @@

- Englist Β· + English Β· δΈ­ζ–‡ Β· Π£ΠΊΡ€Π°Ρ—Π½ΡΡŒΠΊΠ° Β· Česky Β· diff --git a/go.mod b/go.mod index c4bba87944..d9208ccaf4 100644 --- a/go.mod +++ b/go.mod @@ -6,22 +6,21 @@ require ( firebase.google.com/go v3.13.0+incompatible github.com/dtm-labs/rockscache v0.1.1 github.com/gin-gonic/gin v1.9.1 - github.com/go-playground/validator/v10 v10.18.0 + github.com/go-playground/validator/v10 v10.20.0 github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.66-alpha.1 - github.com/openimsdk/tools v0.0.49-alpha.19 + github.com/openimsdk/protocol v0.0.72-alpha.18 + github.com/openimsdk/tools v0.0.50-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 google.golang.org/api v0.165.0 google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -30,12 +29,13 @@ require github.com/google/uuid v1.6.0 require ( github.com/IBM/sarama v1.43.0 github.com/fatih/color v1.14.1 + github.com/gin-contrib/gzip v1.0.1 github.com/go-redis/redis v6.15.9+incompatible github.com/go-redis/redismock/v9 v9.2.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.13 + github.com/openimsdk/gomake v0.0.14-alpha.5 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible @@ -43,6 +43,7 @@ require ( github.com/stathat/consistent v1.0.0 go.uber.org/automaxprocs v1.5.3 golang.org/x/sync v0.7.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( @@ -54,11 +55,31 @@ require ( cloud.google.com/go/longrunning v0.5.4 // indirect cloud.google.com/go/storage v1.36.0 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect + github.com/aws/aws-sdk-go-v2 v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect + github.com/aws/smithy-go v1.17.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -99,8 +120,9 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lithammer/shortuuid v3.0.0+incompatible // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -113,12 +135,13 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/qiniu/go-sdk/v7 v7.18.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -168,11 +191,10 @@ require ( require ( github.com/go-playground/locales v0.14.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/lestrrat-go/strftime v1.0.6 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/spf13/cobra v1.8.0 - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index d64752c951..e51f188145 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,42 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= +github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= +github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= +github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 h1:1w11lfXOa8HoHoSlNtt4mqv/N3HmDOa+OnUH3Y9DHm8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -29,18 +65,20 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= @@ -49,6 +87,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -84,6 +123,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= +github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= @@ -95,14 +136,20 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= -github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= @@ -214,12 +261,19 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= @@ -239,8 +293,8 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -268,16 +322,19 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= -github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.66-alpha.1 h1:/8y+aXQeX6+IgfFxujHbRgJylqJRkwF5gMrwNhWMsiU= -github.com/openimsdk/protocol v0.0.66-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= -github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= +github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/protocol v0.0.72-alpha.18 h1:EytTtgZuXMG1cgTlJryqXXSO1J3t3wrLIn3Os2PRBEE= +github.com/openimsdk/protocol v0.0.72-alpha.18/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo= +github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -294,12 +351,18 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -332,6 +395,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -351,8 +415,8 @@ github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -399,15 +463,17 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -431,10 +497,11 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= @@ -453,20 +520,26 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= @@ -520,11 +593,13 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -533,12 +608,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/api/conversation.go b/internal/api/conversation.go index f273eaa4a7..360313ea87 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -50,3 +50,15 @@ func (o *ConversationApi) SetConversations(c *gin.Context) { func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { a2r.Call(conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client, c) } + +func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client, c) +} + +func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetIncrementalConversation, o.Client, c) +} + +func (o *ConversationApi) GetOwnerConversation(c *gin.Context) { + a2r.Call(conversation.ConversationClient.GetOwnerConversation, o.Client, c) +} diff --git a/internal/api/friend.go b/internal/api/friend.go index 1fea38b313..f9f15fb246 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -16,8 +16,9 @@ package api import ( "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" ) @@ -28,68 +29,83 @@ func NewFriendApi(client rpcclient.Friend) FriendApi { } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.ApplyToAddFriend, o.Client, c) + a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.Call(friend.FriendClient.RespondFriendApply, o.Client, c) + a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.DeleteFriend, o.Client, c) + a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriendsApply, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.Call(friend.FriendClient.SetFriendRemark, o.Client, c) + a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.AddBlack, o.Client, c) + a2r.Call(relation.FriendClient.AddBlack, o.Client, c) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationBlacks, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.RemoveBlack, o.Client, c) + a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.ImportFriends, o.Client, c) + a2r.Call(relation.FriendClient.ImportFriends, o.Client, c) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.IsFriend, o.Client, c) + a2r.Call(relation.FriendClient.IsFriend, o.Client, c) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.Call(friend.FriendClient.GetFriendIDs, o.Client, c) + a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) + a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } + func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) + a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c) +} + +func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) +} + +// GetIncrementalBlacks is temporarily unused. +// Deprecated: This function is currently unused and may be removed in future versions. +func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) +} + +func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { + a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c) } diff --git a/internal/api/group.go b/internal/api/group.go index 6079c53437..14f50cacdd 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -35,6 +35,10 @@ func (o *GroupApi) SetGroupInfo(c *gin.Context) { a2r.Call(group.GroupClient.SetGroupInfo, o.Client, c) } +func (o *GroupApi) SetGroupInfoEX(c *gin.Context) { + a2r.Call(group.GroupClient.SetGroupInfoEX, o.Client, c) +} + func (o *GroupApi) JoinGroup(c *gin.Context) { a2r.Call(group.GroupClient.JoinGroup, o.Client, c) } @@ -65,6 +69,7 @@ func (o *GroupApi) GetGroupUsersReqApplicationList(c *gin.Context) { func (o *GroupApi) GetGroupsInfo(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c) + //a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo)) } func (o *GroupApi) KickGroupMember(c *gin.Context) { @@ -73,6 +78,7 @@ func (o *GroupApi) KickGroupMember(c *gin.Context) { func (o *GroupApi) GetGroupMembersInfo(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c) + //a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo)) } func (o *GroupApi) GetGroupMemberList(c *gin.Context) { @@ -134,3 +140,23 @@ func (o *GroupApi) GetGroups(c *gin.Context) { func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c) } + +func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { + a2r.Call(group.GroupClient.BatchGetIncrementalGroupMember, o.Client, c) +} + +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) +} + +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) +} diff --git a/internal/api/init.go b/internal/api/init.go index 23866c4a07..e83dfc2ea5 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,7 +29,6 @@ import ( "time" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" @@ -72,10 +71,8 @@ func Start(ctx context.Context, index int, config *Config) error { netDone <- struct{}{} return } - p := ginprom.NewPrometheus("app", prommetrics.GetGinCusMetrics("Api")) - p.SetListenAddress(fmt.Sprintf(":%d", prometheusPort)) - if err = p.Use(router); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) + if err := prommetrics.ApiInit(prometheusPort); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } }() diff --git a/internal/api/msg.go b/internal/api/msg.go index 180342e591..bf7cb83a43 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -49,14 +49,14 @@ func NewMessageApi(msgRpcClient *rpcclient.Message, userRpcClient *rpcclient.Use userRpcClient: rpcclient.NewUserRpcClientByUser(userRpcClient), imAdminUserID: imAdminUserID} } -func (MessageApi) SetOptions(options map[string]bool, value bool) { +func (*MessageApi) SetOptions(options map[string]bool, value bool) { datautil.SetSwitchFromOptions(options, constant.IsHistory, value) datautil.SetSwitchFromOptions(options, constant.IsPersistent, value) datautil.SetSwitchFromOptions(options, constant.IsSenderSync, value) datautil.SetSwitchFromOptions(options, constant.IsConversationUpdate, value) } -func (m MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) *msg.SendMsgReq { +func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) *msg.SendMsgReq { var newContent string options := make(map[string]bool, 5) switch params.ContentType { @@ -101,6 +101,7 @@ func (m MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) SendTime: params.SendTime, Options: options, OfflinePushInfo: params.OfflinePushInfo, + Ex: params.Ex, }, } return &pbData @@ -230,7 +231,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { } // Set the status to successful if the message is sent. - var status int = constant.MsgSendSuccessed + var status = constant.MsgSendSuccessed // Attempt to update the message sending status in the system. _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ diff --git a/internal/api/router.go b/internal/api/router.go index 6005671785..3817070b16 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,9 +2,18 @@ package api import ( "fmt" + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "net/http" + "strings" + + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -12,12 +21,30 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "net/http" - "strings" ) +const ( + NoCompression = -1 + DefaultCompression = 0 + BestCompression = 1 + BestSpeed = 2 +) + +func prommetricsGin() gin.HandlerFunc { + return func(c *gin.Context) { + c.Next() + path := c.FullPath() + if c.Writer.Status() == http.StatusNotFound { + prommetrics.HttpCall("<404>", c.Request.Method, c.Writer.Status()) + } else { + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + } + if resp := apiresp.GetGinApiResponse(c); resp != nil { + prommetrics.APICall(path, c.Request.Method, resp.ErrCode) + } + } +} + func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) @@ -35,8 +62,16 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - - r.Use(gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + switch config.API.Api.CompressionLevel { + case NoCompression: + case DefaultCompression: + r.Use(gzip.Gzip(gzip.DefaultCompression)) + case BestCompression: + r.Use(gzip.Gzip(gzip.BestCompression)) + case BestSpeed: + r.Use(gzip.Gzip(gzip.BestSpeed)) + } + r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) userRouterGroup := r.Group("/user") @@ -81,17 +116,21 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/add_black", f.AddBlack) friendRouterGroup.POST("/get_black_list", f.GetPaginationBlacks) friendRouterGroup.POST("/remove_black", f.RemoveBlack) + friendRouterGroup.POST("/get_incremental_blacks", f.GetIncrementalBlacks) friendRouterGroup.POST("/import_friend", f.ImportFriends) friendRouterGroup.POST("/is_friend", f.IsFriend) friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) + friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) + friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") { groupRouterGroup.POST("/create_group", g.CreateGroup) groupRouterGroup.POST("/set_group_info", g.SetGroupInfo) + groupRouterGroup.POST("/set_group_info_ex", g.SetGroupInfoEX) groupRouterGroup.POST("/join_group", g.JoinGroup) groupRouterGroup.POST("/quit_group", g.QuitGroup) groupRouterGroup.POST("/group_application_response", g.ApplicationGroupResponse) @@ -114,6 +153,11 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_groups", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_members", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_members_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") @@ -183,6 +227,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/get_conversations", c.GetConversations) conversationGroup.POST("/set_conversations", c.SetConversations) conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) + conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) + conversationGroup.POST("/get_incremental_conversations", c.GetIncrementalConversation) + conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) } statisticsGroup := r.Group("/statistics") diff --git a/internal/api/user.go b/internal/api/user.go index d48111b9eb..dba7cd3129 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -36,9 +36,11 @@ func (u *UserApi) UserRegister(c *gin.Context) { a2r.Call(user.UserClient.UserRegister, u.Client, c) } +// UpdateUserInfo is deprecated. Use UpdateUserInfoEx func (u *UserApi) UpdateUserInfo(c *gin.Context) { a2r.Call(user.UserClient.UpdateUserInfo, u.Client, c) } + func (u *UserApi) UpdateUserInfoEx(c *gin.Context) { a2r.Call(user.UserClient.UpdateUserInfoEx, u.Client, c) } diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 0581a025b4..bc06fa9507 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -20,6 +20,9 @@ import ( "runtime/debug" "sync" "sync/atomic" + "time" + + "google.golang.org/protobuf/proto" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" @@ -29,7 +32,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/stringutil" - "google.golang.org/protobuf/proto" ) var ( @@ -72,6 +74,10 @@ type Client struct { closed atomic.Bool closedErr error token string + hbCtx context.Context + hbCancel context.CancelFunc + subLock *sync.Mutex + subUserIDs map[string]struct{} // client conn subscription list } // ResetClient updates the client's state with new connection and context information. @@ -88,14 +94,28 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closed.Store(false) c.closedErr = nil c.token = ctx.GetToken() + c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) + c.subLock = new(sync.Mutex) + if c.subUserIDs != nil { + clear(c.subUserIDs) + } + c.subUserIDs = make(map[string]struct{}) } -func (c *Client) pingHandler(_ string) error { +func (c *Client) pingHandler(appData string) error { if err := c.conn.SetReadDeadline(pongWait); err != nil { return err } - return c.writePongMsg() + log.ZDebug(c.ctx, "ping Handler Success.", "appData", appData) + return c.writePongMsg(appData) +} + +func (c *Client) pongHandler(_ string) error { + if err := c.conn.SetReadDeadline(pongWait); err != nil { + return err + } + return nil } // readMessage continuously reads messages from the connection. @@ -110,7 +130,9 @@ func (c *Client) readMessage() { c.conn.SetReadLimit(maxMessageSize) _ = c.conn.SetReadDeadline(pongWait) + c.conn.SetPongHandler(c.pongHandler) c.conn.SetPingHandler(c.pingHandler) + c.activeHeartbeat(c.hbCtx) for { log.ZDebug(c.ctx, "readMessage") @@ -141,12 +163,13 @@ func (c *Client) readMessage() { return case PingMessage: - err := c.writePongMsg() + err := c.writePongMsg("") log.ZError(c.ctx, "writePongMsg", err) case CloseMessage: c.closedErr = ErrClientClosed return + default: } } @@ -198,10 +221,16 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.SendSignalMessage(ctx, binaryReq) case WSPullMsgBySeqList: resp, messageErr = c.longConnServer.PullMessageBySeqList(ctx, binaryReq) + case WSPullMsg: + resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) + case WSGetConvMaxReadSeq: + resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) + case WsSubUserOnlineStatus: + resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", @@ -226,15 +255,14 @@ func (c *Client) setAppBackgroundStatus(ctx context.Context, req *Req) ([]byte, } func (c *Client) close() { + c.w.Lock() + defer c.w.Unlock() if c.closed.Load() { return } - - c.w.Lock() - defer c.w.Unlock() - c.closed.Store(true) c.conn.Close() + c.hbCancel() // Close server-initiated heartbeat. c.longConnServer.UnRegister(c) } @@ -248,11 +276,13 @@ func (c *Client) replyMessage(ctx context.Context, binaryReq *Req, err error, re ErrMsg: errResp.ErrMsg, Data: resp, } + t := time.Now() log.ZDebug(ctx, "gateway reply message", "resp", mReply.String()) err = c.writeBinaryMsg(mReply) if err != nil { log.ZWarn(ctx, "wireBinaryMsg replyMessage", err, "resp", mReply.String()) } + log.ZDebug(ctx, "wireBinaryMsg end", "time cost", time.Since(t)) if binaryReq.ReqIdentifier == WsLogoutMsg { return errs.New("user logout", "operationID", binaryReq.OperationID).Wrap() @@ -292,6 +322,14 @@ func (c *Client) KickOnlineMessage() error { return err } +func (c *Client) PushUserOnlineStatus(data []byte) error { + resp := Resp{ + ReqIdentifier: WsSubUserOnlineStatus, + Data: data, + } + return c.writeBinaryMsg(resp) +} + func (c *Client) writeBinaryMsg(resp Resp) error { if c.closed.Load() { return nil @@ -321,7 +359,29 @@ func (c *Client) writeBinaryMsg(resp Resp) error { return c.conn.WriteMessage(MessageBinary, encodedBuf) } -func (c *Client) writePongMsg() error { +// Actively initiate Heartbeat when platform in Web. +func (c *Client) activeHeartbeat(ctx context.Context) { + if c.PlatformID == constant.WebPlatformID { + go func() { + log.ZDebug(ctx, "server initiative send heartbeat start.") + ticker := time.NewTicker(pingPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := c.writePingMsg(); err != nil { + log.ZWarn(c.ctx, "send Ping Message error.", err) + return + } + case <-c.hbCtx.Done(): + return + } + } + }() + } +} +func (c *Client) writePingMsg() error { if c.closed.Load() { return nil } @@ -334,5 +394,28 @@ func (c *Client) writePongMsg() error { return err } - return c.conn.WriteMessage(PongMessage, nil) + return c.conn.WriteMessage(PingMessage, nil) +} + +func (c *Client) writePongMsg(appData string) error { + log.ZDebug(c.ctx, "write Pong Msg in Server", "appData", appData) + if c.closed.Load() { + log.ZWarn(c.ctx, "is closed in server", nil, "appdata", appData, "closed err", c.closedErr) + return nil + } + + c.w.Lock() + defer c.w.Unlock() + + err := c.conn.SetWriteDeadline(writeWait) + if err != nil { + log.ZWarn(c.ctx, "SetWriteDeadline in Server have error", errs.Wrap(err), "writeWait", writeWait, "appData", appData) + return errs.Wrap(err) + } + err = c.conn.WriteMessage(PongMessage, []byte(appData)) + if err != nil { + log.ZWarn(c.ctx, "Write Message have error", errs.Wrap(err), "Pong msg", PongMessage) + } + + return errs.Wrap(err) } diff --git a/internal/msggateway/compressor_test.go b/internal/msggateway/compressor_test.go index 173c9bb204..952bd4d95b 100644 --- a/internal/msggateway/compressor_test.go +++ b/internal/msggateway/compressor_test.go @@ -16,10 +16,10 @@ package msggateway import ( "crypto/rand" + "github.com/stretchr/testify/assert" "sync" "testing" - - "github.com/stretchr/testify/assert" + "unsafe" ) func mockRandom() []byte { @@ -132,3 +132,8 @@ func BenchmarkDecompressWithSyncPool(b *testing.B) { assert.Equal(b, nil, err) } } + +func TestName(t *testing.T) { + t.Log(unsafe.Sizeof(Client{})) + +} diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 64664ac0ab..584cebe1e1 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -39,10 +39,13 @@ const ( WSPullMsgBySeqList = 1002 WSSendMsg = 1003 WSSendSignalMsg = 1004 + WSPullMsg = 1005 + WSGetConvMaxReadSeq = 1006 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 + WsSubUserOnlineStatus = 2005 WSDataError = 3001 ) @@ -53,6 +56,9 @@ const ( // Time allowed to read the next pong message from the peer. pongWait = 30 * time.Second + // Send pings to peer with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + // Maximum message size allowed from peer. maxMessageSize = 51200 ) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8ff6d10018..28c227162a 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -19,18 +19,27 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mq/memamq" + "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" + "sync/atomic" ) func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + if s.ready != nil { + return s.ready(s) + } return nil } @@ -50,18 +59,23 @@ type Server struct { LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} + ready func(srv *Server) error + userRcp rpcclient.UserRpcClient + queue *memamq.MemoryQueue } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config) *Server { +func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, + ready: ready, + queue: memamq.NewMemoryQueue(512, 1024*16), } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} @@ -117,55 +131,93 @@ func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.Onli return nil, nil } -func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq, -) (*msggateway.OnlineBatchPushOneMsgResp, error) { - var singleUserResults []*msggateway.SingleMsgToUserResults - for _, v := range req.PushToUserIDs { - var resp []*msggateway.SingleMsgToUserPlatform - results := &msggateway.SingleMsgToUserResults{ - UserID: v, +func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { + clients, ok := s.LongConnServer.GetUserAllCons(userID) + if !ok { + log.ZDebug(ctx, "push user not online", "userID", userID) + return &msggateway.SingleMsgToUserResults{ + UserID: userID, } - clients, ok := s.LongConnServer.GetUserAllCons(v) - if !ok { - log.ZDebug(ctx, "push user not online", "userID", v) - results.Resp = resp - singleUserResults = append(singleUserResults, results) + } + log.ZDebug(ctx, "push user online", "clients", clients, "userID", userID) + result := &msggateway.SingleMsgToUserResults{ + UserID: userID, + Resp: make([]*msggateway.SingleMsgToUserPlatform, 0, len(clients)), + } + for _, client := range clients { + if client == nil { continue } - - log.ZDebug(ctx, "push user online", "clients", clients, "userID", v) - for _, client := range clients { - if client == nil { - continue + userPlatform := &msggateway.SingleMsgToUserPlatform{ + RecvPlatFormID: int32(client.PlatformID), + } + if !client.IsBackground || + (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { + err := client.PushMessage(ctx, msgData) + if err != nil { + userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) + } else { + if _, ok := s.pushTerminal[client.PlatformID]; ok { + result.OnlinePush = true + } } + } else { + userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) + } + result.Resp = append(result.Resp, userPlatform) + } + return result +} - userPlatform := &msggateway.SingleMsgToUserPlatform{ - RecvPlatFormID: int32(client.PlatformID), +func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { + if len(req.PushToUserIDs) == 0 { + return &msggateway.OnlineBatchPushOneMsgResp{}, nil + } + ch := make(chan *msggateway.SingleMsgToUserResults, len(req.PushToUserIDs)) + var count atomic.Int64 + count.Add(int64(len(req.PushToUserIDs))) + for i := range req.PushToUserIDs { + userID := req.PushToUserIDs[i] + err := s.queue.PushCtx(ctx, func() { + ch <- s.pushToUser(ctx, userID, req.MsgData) + if count.Add(-1) == 0 { + close(ch) } - if !client.IsBackground || - (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { - err := client.PushMessage(ctx, req.MsgData) - if err != nil { - userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) - resp = append(resp, userPlatform) - } else { - if _, ok := s.pushTerminal[client.PlatformID]; ok { - results.OnlinePush = true - resp = append(resp, userPlatform) - } - } - } else { - userPlatform.ResultCode = int64(servererrs.ErrIOSBackgroundPushErr.Code()) - resp = append(resp, userPlatform) + }) + if err != nil { + if count.Add(-1) == 0 { + close(ch) + } + log.ZError(ctx, "pushToUser MemoryQueue failed", err, "userID", userID) + ch <- &msggateway.SingleMsgToUserResults{ + UserID: userID, } } - results.Resp = resp - singleUserResults = append(singleUserResults, results) } - - return &msggateway.OnlineBatchPushOneMsgResp{ - SinglePushResult: singleUserResults, - }, nil + resp := &msggateway.OnlineBatchPushOneMsgResp{ + SinglePushResult: make([]*msggateway.SingleMsgToUserResults, 0, len(req.PushToUserIDs)), + } + for { + select { + case <-ctx.Done(): + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ctx done", context.Cause(ctx)) + userIDSet := datautil.SliceSet(req.PushToUserIDs) + for _, results := range resp.SinglePushResult { + delete(userIDSet, results.UserID) + } + for userID := range userIDSet { + resp.SinglePushResult = append(resp.SinglePushResult, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return resp, nil + case res, ok := <-ch: + if !ok { + return resp, nil + } + resp.SinglePushResult = append(resp.SinglePushResult, res) + } + } } func (s *Server) KickUserOffline( diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index f4d8b0381a..50da060976 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,8 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" + "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -26,6 +28,7 @@ import ( type Config struct { MsgGateway config.MsgGateway Share config.Share + RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery } @@ -42,18 +45,25 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } - longServer, err := NewWsServer( + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) + if err != nil { + return err + } + longServer := NewWsServer( conf, WithPort(wsPort), WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)), WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second), WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - if err != nil { - return err - } - hubServer := NewServer(rpcPort, longServer, conf) + hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) + return nil + }) + + go longServer.ChangeOnlineStatus(4) + netDone := make(chan error) go func() { err = hubServer.Start(ctx, index, conf) diff --git a/internal/msggateway/long_conn.go b/internal/msggateway/long_conn.go index 7d5bef4c3a..c1b3e27c93 100644 --- a/internal/msggateway/long_conn.go +++ b/internal/msggateway/long_conn.go @@ -16,10 +16,11 @@ package msggateway import ( "encoding/json" - "github.com/openimsdk/tools/apiresp" "net/http" "time" + "github.com/openimsdk/tools/apiresp" + "github.com/gorilla/websocket" "github.com/openimsdk/tools/errs" ) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 8a11e6ab3c..4b78c10048 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -19,6 +19,8 @@ import ( "sync" "github.com/go-playground/validator/v10" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/msg" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/jsonutil" - "google.golang.org/protobuf/proto" ) type Req struct { @@ -94,6 +95,8 @@ type MessageHandler interface { SendMessage(context context.Context, data *Req) ([]byte, error) SendSignalMessage(context context.Context, data *Req) ([]byte, error) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) + GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) + GetSeqMessage(context context.Context, data *Req) ([]byte, error) UserLogout(context context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(context context.Context, data *Req) ([]byte, bool, error) } @@ -175,7 +178,7 @@ func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]by func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) { req := sdkws.PullMessageBySeqsReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { - return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") } if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "PullMessageBySeqsReq") @@ -191,6 +194,44 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ return c, nil } +func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) { + req := msg.GetConversationsHasReadAndMaxSeqReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetConversationsHasReadAndMaxSeq") + } + resp, err := g.msgRpcClient.GetConversationsHasReadAndMaxSeq(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + return c, nil +} + +func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, error) { + req := msg.GetSeqMessageReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "GetSeqMessage") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetSeqMessage") + } + resp, err := g.msgRpcClient.GetSeqMessage(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetSeqMessage") + } + return c, nil +} + func (g GrpcHandler) UserLogout(context context.Context, data *Req) ([]byte, error) { req := push.DelUserPushTokenReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go new file mode 100644 index 0000000000..27b4544aaa --- /dev/null +++ b/internal/msggateway/online.go @@ -0,0 +1,117 @@ +package msggateway + +import ( + "context" + "crypto/md5" + "encoding/binary" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" + "math/rand" + "os" + "strconv" + "sync/atomic" + "time" +) + +func (ws *WsServer) ChangeOnlineStatus(concurrent int) { + if concurrent < 1 { + concurrent = 1 + } + const renewalTime = cachekey.OnlineExpire / 3 + //const renewalTime = time.Second * 10 + renewalTicker := time.NewTicker(renewalTime) + + requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) + changeStatus := make([][]UserState, concurrent) + + for i := 0; i < concurrent; i++ { + requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) + changeStatus[i] = make([]UserState, 0, 100) + } + + mergeTicker := time.NewTicker(time.Second) + + local2pb := func(u UserState) *pbuser.UserOnlineStatus { + return &pbuser.UserOnlineStatus{ + UserID: u.UserID, + Online: u.Online, + Offline: u.Offline, + } + } + + rNum := rand.Uint64() + pushUserState := func(us ...UserState) { + for _, u := range us { + sum := md5.Sum([]byte(u.UserID)) + i := (binary.BigEndian.Uint64(sum[:]) + rNum) % uint64(concurrent) + changeStatus[i] = append(changeStatus[i], u) + status := changeStatus[i] + if len(status) == cap(status) { + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[:0] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + } + + pushAllUserState := func() { + for i, status := range changeStatus { + if len(status) == 0 { + continue + } + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[:0] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + + var count atomic.Int64 + operationIDPrefix := fmt.Sprintf("p_%d_", os.Getpid()) + doRequest := func(req *pbuser.SetUserOnlineStatusReq) { + opIdCtx := mcontext.SetOperationID(context.Background(), operationIDPrefix+strconv.FormatInt(count.Add(1), 10)) + ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) + defer cancel() + if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { + log.ZError(ctx, "update user online status", err) + } + } + + for i := 0; i < concurrent; i++ { + go func(ch <-chan *pbuser.SetUserOnlineStatusReq) { + for req := range ch { + doRequest(req) + } + }(requestChs[i]) + } + + for { + select { + case <-mergeTicker.C: + pushAllUserState() + case now := <-renewalTicker.C: + deadline := now.Add(-cachekey.OnlineExpire / 3) + users := ws.clients.GetAllUserStatus(deadline, now) + log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users), "users", users) + pushUserState(users...) + case state := <-ws.clients.UserState(): + log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) + pushUserState(state) + } + } +} diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go new file mode 100644 index 0000000000..9bb41e0dfd --- /dev/null +++ b/internal/msggateway/subscription.go @@ -0,0 +1,166 @@ +package msggateway + +import ( + "context" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + "google.golang.org/protobuf/proto" + "sync" +) + +func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + } + ws.pushUserIDOnlineStatus(ctx, userID, platformIDs) +} + +func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) { + var sub sdkws.SubUserOnlineStatus + if err := proto.Unmarshal(data.Data, &sub); err != nil { + return nil, err + } + ws.subscription.Sub(client, sub.SubscribeUserID, sub.UnsubscribeUserID) + var resp sdkws.SubUserOnlineStatusTips + if len(sub.SubscribeUserID) > 0 { + resp.Subscribers = make([]*sdkws.SubUserOnlineStatusElem, 0, len(sub.SubscribeUserID)) + for _, userID := range sub.SubscribeUserID { + platformIDs, err := ws.online.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + resp.Subscribers = append(resp.Subscribers, &sdkws.SubUserOnlineStatusElem{ + UserID: userID, + OnlinePlatformIDs: platformIDs, + }) + } + } + return proto.Marshal(&resp) +} + +func newSubscription() *Subscription { + return &Subscription{ + userIDs: make(map[string]*subClient), + } +} + +type subClient struct { + clients map[string]*Client +} + +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient // subscribe to the user's client connection +} + +func (s *Subscription) DelClient(client *Client) { + client.subLock.Lock() + userIDs := datautil.Keys(client.subUserIDs) + for _, userID := range userIDs { + delete(client.subUserIDs, userID) + } + client.subLock.Unlock() + if len(userIDs) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for _, userID := range userIDs { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } +} + +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + +func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { + if len(addUserIDs)+len(delUserIDs) == 0 { + return + } + var ( + del = make(map[string]struct{}) + add = make(map[string]struct{}) + ) + client.subLock.Lock() + for _, userID := range delUserIDs { + if _, ok := client.subUserIDs[userID]; !ok { + continue + } + del[userID] = struct{}{} + delete(client.subUserIDs, userID) + } + for _, userID := range addUserIDs { + delete(del, userID) + if _, ok := client.subUserIDs[userID]; ok { + continue + } + client.subUserIDs[userID] = struct{}{} + add[userID] = struct{}{} + } + client.subLock.Unlock() + if len(del)+len(add) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for userID := range del { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } + for userID := range add { + sub, ok := s.userIDs[userID] + if !ok { + sub = &subClient{clients: make(map[string]*Client)} + s.userIDs[userID] = sub + } + sub.clients[addr] = client + } +} + +func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { + clients := ws.subscription.GetClient(userID) + if len(clients) == 0 { + return + } + onlineStatus, err := proto.Marshal(&sdkws.SubUserOnlineStatusTips{ + Subscribers: []*sdkws.SubUserOnlineStatusElem{{UserID: userID, OnlinePlatformIDs: platformIDs}}, + }) + if err != nil { + log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) + return + } + for _, client := range clients { + if err := client.PushUserOnlineStatus(onlineStatus); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "changePlatformID", platformIDs) + } + } +} diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 79cc53d1bc..5baa4f9012 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,135 +1,185 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package msggateway import ( - "context" - "sync" - - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" + "sync" + "time" ) -type UserMap struct { - m sync.Map +type UserMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState + RecvSubChange(userID string, platformIDs []int32) bool } -func newUserMap() *UserMap { - return &UserMap{} +type UserState struct { + UserID string + Online []int32 + Offline []int32 } -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.([]*Client), ok - } - return nil, ok +type UserPlatform struct { + Time time.Time + Clients []*Client } -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.([]*Client) { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, userExisted, true - } - return clients, userExisted, false +func (u *UserPlatform) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) } - return nil, userExisted, false + return platformIDs } -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.([]*Client) - oldClients = append(oldClients, v) - u.m.Store(key, oldClients) - } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) +func (u *UserPlatform) PlatformIDSet() map[int32]struct{} { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make(map[int32]struct{}) + for _, client := range u.Clients { + platformIDs[int32(client.PlatformID)] = struct{}{} + } + return platformIDs +} - var clients []*Client - clients = append(clients, v) - u.m.Store(key, clients) +func newUserMap() UserMap { + return &userMap{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), } } -func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) { - // Attempt to load the clients associated with the key. - allClients, existed := u.m.Load(key) - if !existed { - // Return false immediately if the key does not exist. +type userMap struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState +} + +func (u *userMap) RecvSubChange(userID string, platformIDs []int32) bool { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { return false } - - // Convert allClients to a slice of *Client. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - // Keep clients that do not match the connRemoteAddr. - if client.ctx.GetRemoteAddr() != connRemoteAddr { - remainingClients = append(remainingClients, client) - } + localPlatformIDs := result.PlatformIDSet() + for _, platformID := range platformIDs { + delete(localPlatformIDs, platformID) } + if len(localPlatformIDs) == 0 { + return false + } + u.push(userID, result, nil) + return true +} - // If no clients remain after filtering, delete the key from the map. - if len(remainingClients) == 0 { - u.m.Delete(key) +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() return true + default: + return false } +} - // Otherwise, update the key with the remaining clients. - u.m.Store(key, remainingClients) - return false +func (u *userMap) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false + } + return result.Clients, true } -func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. - return false +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + return clients, true, len(clients) > 0 +} - // Filter out clients that are in the deleteMap. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) +func (u *userMap) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients = append(result.Clients, client) + } else { + result = &UserPlatform{ + Clients: []*Client{client}, } + u.data[userID] = result } + u.push(client.UserID, result, nil) +} - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { + return false } + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, delCli := deleteAddr[client.ctx.GetRemoteAddr()]; delCli { + offline = append(offline, int32(client.PlatformID)) + } else { + result.Clients = append(result.Clients, client) + } + } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} - u.m.Store(key, remainingClients) - return false +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) (result []UserState) { + u.lock.RLock() + defer u.lock.RUnlock() + result = make([]UserState, 0, len(u.data)) + for userID, userPlatform := range u.data { + if deadline.Before(userPlatform.Time) { + continue + } + userPlatform.Time = nowtime + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) + } + return result } -func (u *UserMap) DeleteAll(key string) { - u.m.Delete(key) +func (u *userMap) UserState() <-chan UserState { + return u.ch } diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/ws_server.go similarity index 87% rename from internal/msggateway/n_ws_server.go rename to internal/msggateway/ws_server.go index defec16df1..81392897bf 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/ws_server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/mcontext" "net/http" @@ -48,6 +49,7 @@ type LongConnServer interface { KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) + SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor Encoder MessageHandler @@ -60,7 +62,9 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients *UserMap + clients UserMap + online *rpccache.OnlineCache + subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -90,18 +94,18 @@ func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, ws.disCov = disCov } -func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { - err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) - if err != nil { - log.ZWarn(ctx, "SetUserStatus err", err) - } - switch status { - case constant.Online: - ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) - case constant.Offline: - ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) - } -} +//func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { +// err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) +// if err != nil { +// log.ZWarn(ctx, "SetUserStatus err", err) +// } +// switch status { +// case constant.Online: +// ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) +// case constant.Offline: +// ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) +// } +//} func (ws *WsServer) UnRegister(c *Client) { ws.unregisterChan <- c @@ -119,11 +123,13 @@ func (ws *WsServer) GetUserPlatformCons(userID string, platform int) ([]*Client, return ws.clients.Get(userID, platform) } -func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { +func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { var config configs for _, o := range opts { o(&config) } + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + v := validator.New() return &WsServer{ msgGatewayConfig: msgGatewayConfig, @@ -141,10 +147,11 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), + subscription: newSubscription(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), - }, nil + } } func (ws *WsServer) Run(done chan error) error { @@ -258,7 +265,7 @@ func (ws *WsServer) registerClient(client *Client) { if clientOK { ws.clients.Set(client.UserID, client) // There is already a connection to the platform - log.ZInfo(client.ctx, "repeat login", "userID", client.UserID, "platformID", + log.ZDebug(client.ctx, "repeat login", "userID", client.UserID, "platformID", client.PlatformID, "old remote addr", getRemoteAdders(oldClients)) ws.onlineUserConnNum.Add(1) } else { @@ -268,7 +275,7 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} - log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", ws.msgGatewayConfig.Discovery.Enable) + log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", "discoveryEnable", ws.msgGatewayConfig.Discovery.Enable) if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) @@ -278,15 +285,15 @@ func (ws *WsServer) registerClient(client *Client) { }() } - wg.Add(1) - go func() { - defer wg.Done() - ws.SetUserOnlineStatus(client.ctx, client, constant.Online) - }() + //wg.Add(1) + //go func() { + // defer wg.Done() + // ws.SetUserOnlineStatus(client.ctx, client, constant.Online) + //}() wg.Wait() - log.ZInfo( + log.ZDebug( client.ctx, "user online", "online user Num", @@ -309,7 +316,7 @@ func getRemoteAdders(client []*Client) string { } func (ws *WsServer) KickUserConn(client *Client) error { - ws.clients.deleteClients(client.UserID, []*Client{client}) + ws.clients.DeleteClients(client.UserID, []*Client{client}) return client.KickOnlineMessage() } @@ -325,7 +332,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if !clientOK { return } - ws.clients.deleteClients(newClient.UserID, oldClients) + ws.clients.DeleteClients(newClient.UserID, oldClients) for _, c := range oldClients { err := c.KickOnlineMessage() if err != nil { @@ -345,14 +352,15 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.DeleteClients(client.UserID, []*Client{client}) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) - log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", + ws.subscription.DelClient(client) + //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) + log.ZDebug(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), ) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 65d04f3810..7dc2ebeea0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -16,29 +16,28 @@ package msgtransfer import ( "context" + "errors" "fmt" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" - "net/http" - "os" - "os/signal" - "syscall" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -67,6 +66,7 @@ type Config struct { func Start(ctx context.Context, index int, config *Config) error { log.CInfo(ctx, "MSG-TRANSFER server is initializing", "prometheusPorts", config.MsgTransfer.Prometheus.Ports, "index", index) + mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -75,32 +75,43 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return err } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err } - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgDatabase, &conversationRpcClient, &groupRpcClient) + historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) if err != nil { return err } - historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgDatabase) + historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgTransferDatabase) if err != nil { return err } + msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, @@ -130,14 +141,8 @@ func (m *MsgTransfer) Start(index int, config *Config) error { netDone <- struct{}{} return } - proreg := prometheus.NewRegistry() - proreg.MustRegister( - collectors.NewGoCollector(), - ) - proreg.MustRegister(prommetrics.GetGrpcCusMetrics("Transfer", &config.Share)...) - http.Handle("/metrics", promhttp.HandlerFor(proreg, promhttp.HandlerOpts{Registry: proreg})) - err = http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), nil) - if err != nil && err != http.ErrServerClosed { + + if err := prommetrics.TransferInit(prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index d671ec52a2..b0078649cb 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -16,6 +16,12 @@ package msgtransfer import ( "context" + "encoding/json" + "errors" + "strconv" + "strings" + "time" + "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -31,9 +37,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/stringutil" "google.golang.org/protobuf/proto" - "strconv" - "strings" - "time" ) const ( @@ -54,19 +57,19 @@ type OnlineHistoryRedisConsumerHandler struct { redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase conversationRpcClient *rpcclient.ConversationRpcClient groupRpcClient *rpcclient.GroupRpcClient } -func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase, +func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*OnlineHistoryRedisConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) if err != nil { return nil, err } var och OnlineHistoryRedisConsumerHandler - och.msgDatabase = database + och.msgTransferDatabase = database b := batcher.New[sarama.ConsumerMessage]( batcher.WithSize(size), @@ -88,6 +91,7 @@ func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database cont och.conversationRpcClient = conversationRpcClient och.groupRpcClient = groupRpcClient och.historyConsumerGroup = historyConsumerGroup + return &och, err } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { @@ -96,6 +100,7 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID ctx = withAggregationCtx(ctx, ctxMessages) log.ZInfo(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMessages), "key", val.Key()) + och.doSetReadSeq(ctx, ctxMessages) storageMsgList, notStorageMsgList, storageNotificationList, notStorageNotificationList := och.categorizeMessageLists(ctxMessages) @@ -109,6 +114,60 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID och.handleNotification(ctx, val.Key(), conversationIDNotification, storageNotificationList, notStorageNotificationList) } +func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, msgs []*ContextMsg) { + type seqKey struct { + conversationID string + userID string + } + var readSeq map[seqKey]int64 + for _, msg := range msgs { + if msg.message.ContentType != constant.HasReadReceipt { + continue + } + var elem sdkws.NotificationElem + if err := json.Unmarshal(msg.message.Content, &elem); err != nil { + log.ZError(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) + continue + } + var tips sdkws.MarkAsReadTips + if err := json.Unmarshal([]byte(elem.Detail), &tips); err != nil { + log.ZError(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) + continue + } + if len(tips.Seqs) > 0 { + for _, seq := range tips.Seqs { + if tips.HasReadSeq < seq { + tips.HasReadSeq = seq + } + } + clear(tips.Seqs) + tips.Seqs = nil + } + if tips.HasReadSeq < 0 { + continue + } + if readSeq == nil { + readSeq = make(map[seqKey]int64) + } + key := seqKey{ + conversationID: tips.ConversationID, + userID: tips.MarkAsReadUserID, + } + if readSeq[key] > tips.HasReadSeq { + continue + } + readSeq[key] = tips.HasReadSeq + } + if readSeq == nil { + return + } + for key, seq := range readSeq { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { + log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) + } + } +} + func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg { var ctxMessages []*ContextMsg for i := 0; i < len(consumerMessages); i++ { @@ -179,6 +238,11 @@ func (och *OnlineHistoryRedisConsumerHandler) categorizeMessageLists(totalMsgs [ } func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key, conversationID string, storageList, notStorageList []*ContextMsg) { + log.ZInfo(ctx, "handle storage msg") + for _, storageMsg := range storageList { + log.ZDebug(ctx, "handle storage msg", "msg", storageMsg.message.String()) + } + och.toPushTopic(ctx, key, conversationID, notStorageList) var storageMessageList []*sdkws.MsgData for _, msg := range storageList { @@ -186,21 +250,25 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } if len(storageMessageList) > 0 { msg := storageMessageList[0] - lastSeq, isNewConversation, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) - if err != nil && errs.Unwrap(err) != redis.Nil { + lastSeq, isNewConversation, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } + log.ZInfo(ctx, "BatchInsertChat2Cache end") + if isNewConversation { switch msg.SessionType { case constant.ReadGroupChatType: - log.ZInfo(ctx, "group chat first create conversation", "conversationID", + log.ZDebug(ctx, "group chat first create conversation", "conversationID", conversationID) userIDs, err := och.groupRpcClient.GetGroupMemberIDs(ctx, msg.GroupID) if err != nil { log.ZWarn(ctx, "get group member ids error", err, "conversationID", conversationID) } else { + log.ZInfo(ctx, "GetGroupMemberIDs end") + if err := och.conversationRpcClient.GroupChatFirstCreateConversation(ctx, msg.GroupID, userIDs); err != nil { log.ZWarn(ctx, "single chat first create conversation error", err, @@ -219,13 +287,16 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } } - log.ZDebug(ctx, "success incr to next topic") - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + log.ZInfo(ctx, "success incr to next topic") + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) } + log.ZInfo(ctx, "MsgToMongoMQ end") + och.toPushTopic(ctx, key, conversationID, storageList) + log.ZInfo(ctx, "toPushTopic end") } } @@ -237,14 +308,14 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con storageMessageList = append(storageMessageList, msg.message) } if len(storageMessageList) > 0 { - lastSeq, _, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil { log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageMessageList) return } log.ZDebug(ctx, "success to next topic", "conversationID", conversationID) - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) @@ -253,9 +324,10 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con } } -func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(_ context.Context, key, conversationID string, msgs []*ContextMsg) { +func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { - och.msgDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) + log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) + _, _, _ = och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) } } @@ -280,7 +352,7 @@ func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSess func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - log.ZInfo(context.Background(), "online new session msg come", "highWaterMarkOffset", + log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) och.redisMessageBatches.OnComplete = func(lastMessage *sarama.ConsumerMessage, totalCount int) { session.MarkMessage(lastMessage, "") diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index e5651012c6..82002c26b9 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -29,10 +29,10 @@ import ( type OnlineHistoryMongoConsumerHandler struct { historyConsumerGroup *kafka.MConsumerGroup - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase } -func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase) (*OnlineHistoryMongoConsumerHandler, error) { +func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryMongoConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToMongoGroupID, []string{kafkaConf.ToMongoTopic}, true) if err != nil { return nil, err @@ -40,7 +40,7 @@ func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database cont mc := &OnlineHistoryMongoConsumerHandler{ historyConsumerGroup: historyConsumerGroup, - msgDatabase: database, + msgTransferDatabase: database, } return mc, nil } @@ -57,8 +57,8 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont log.ZError(ctx, "msgFromMQ.MsgData is empty", nil, "cMsg", cMsg) return } - log.ZInfo(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) - err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) + log.ZDebug(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) + err = mc.msgTransferDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) if err != nil { log.ZError( ctx, @@ -77,7 +77,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont for _, msg := range msgFromMQ.MsgData { seqs = append(seqs, msg.Seq) } - err = mc.msgDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) + err = mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) if err != nil { log.ZError( ctx, @@ -91,13 +91,13 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont } } -func (OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } -func (OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, -) error { // a instance in the consumer group +) error { // an instance in the consumer group log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) for msg := range claim.Messages() { diff --git a/internal/push/a_test.go b/internal/push/a_test.go new file mode 100644 index 0000000000..8b2d864071 --- /dev/null +++ b/internal/push/a_test.go @@ -0,0 +1,29 @@ +package push + +import ( + "github.com/openimsdk/protocol/sdkws" + "testing" +) + +func TestName(t *testing.T) { + var c ConsumerHandler + c.readCh = make(chan *sdkws.MarkAsReadTips) + + go c.loopRead() + + go func() { + for i := 0; ; i++ { + seq := int64(i + 1) + if seq%3 == 0 { + seq = 1 + } + c.readCh <- &sdkws.MarkAsReadTips{ + ConversationID: "c100", + MarkAsReadUserID: "u100", + HasReadSeq: seq, + } + } + }() + + select {} +} diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 028e7edd34..09831cabfa 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -17,6 +17,7 @@ package dummy import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/tools/log" ) func NewClient() *Dummy { @@ -27,5 +28,6 @@ type Dummy struct { } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { + log.ZDebug(ctx, "dummy push") return nil } diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index 27b19e8fe0..e266f9c464 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -18,11 +18,11 @@ import ( "context" "crypto/sha256" "encoding/hex" - "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "strconv" "sync" "time" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/errs" @@ -91,6 +91,15 @@ func (g *Client) Push(ctx context.Context, userIDs []string, title, content stri for i, v := range s.GetSplitResult() { go func(index int, userIDs []string) { defer wg.Done() + for i := 0; i < len(userIDs); i += maxNum { + end := i + maxNum + if end > len(userIDs) { + end = len(userIDs) + } + if err = g.batchPush(ctx, token, userIDs[i:end], pushReq); err != nil { + log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) + } + } if err = g.batchPush(ctx, token, userIDs, pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) } diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go new file mode 100644 index 0000000000..bf69aed3e2 --- /dev/null +++ b/internal/push/offlinepush_handler.go @@ -0,0 +1,122 @@ +package push + +import ( + "context" + + "github.com/IBM/sarama" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/protocol/constant" + pbpush "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/utils/jsonutil" + "google.golang.org/protobuf/proto" +) + +type OfflinePushConsumerHandler struct { + OfflinePushConsumerGroup *kafka.MConsumerGroup + offlinePusher offlinepush.OfflinePusher +} + +func NewOfflinePushConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher) (*OfflinePushConsumerHandler, error) { + var offlinePushConsumerHandler OfflinePushConsumerHandler + var err error + offlinePushConsumerHandler.offlinePusher = offlinePusher + offlinePushConsumerHandler.OfflinePushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToOfflineGroupID, + []string{config.KafkaConfig.ToOfflinePushTopic}, true) + if err != nil { + return nil, err + } + return &offlinePushConsumerHandler, nil +} + +func (*OfflinePushConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } +func (*OfflinePushConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } +func (o *OfflinePushConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + ctx := o.OfflinePushConsumerGroup.GetContextFromMsg(msg) + o.handleMsg2OfflinePush(ctx, msg.Value) + sess.MarkMessage(msg, "") + } + return nil +} + +func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, msg []byte) { + offlinePushMsg := pbpush.PushMsgReq{} + if err := proto.Unmarshal(msg, &offlinePushMsg); err != nil { + log.ZError(ctx, "offline push Unmarshal msg err", err, "msg", string(msg)) + return + } + if offlinePushMsg.MsgData == nil || offlinePushMsg.UserIDs == nil { + log.ZError(ctx, "offline push msg is empty", errs.New("offlinePushMsg is empty"), "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + return + } + log.ZInfo(ctx, "receive to OfflinePush MQ", "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + + err := o.offlinePushMsg(ctx, offlinePushMsg.MsgData, offlinePushMsg.UserIDs) + if err != nil { + log.ZWarn(ctx, "offline push failed", err, "msg", offlinePushMsg.String()) + } +} + +func (o *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, content string, opts *options.Opts, err error) { + type AtTextElem struct { + Text string `json:"text,omitempty"` + AtUserList []string `json:"atUserList,omitempty"` + IsAtSelf bool `json:"isAtSelf"` + } + + opts = &options.Opts{Signal: &options.Signal{}} + if msg.OfflinePushInfo != nil { + opts.IOSBadgeCount = msg.OfflinePushInfo.IOSBadgeCount + opts.IOSPushSound = msg.OfflinePushInfo.IOSPushSound + opts.Ex = msg.OfflinePushInfo.Ex + } + + if msg.OfflinePushInfo != nil { + title = msg.OfflinePushInfo.Title + content = msg.OfflinePushInfo.Desc + } + if title == "" { + switch msg.ContentType { + case constant.Text: + fallthrough + case constant.Picture: + fallthrough + case constant.Voice: + fallthrough + case constant.Video: + fallthrough + case constant.File: + title = constant.ContentType2PushContent[int64(msg.ContentType)] + case constant.AtText: + ac := AtTextElem{} + _ = jsonutil.JsonStringToStruct(string(msg.Content), &ac) + case constant.SignalingNotification: + title = constant.ContentType2PushContent[constant.SignalMsg] + default: + title = constant.ContentType2PushContent[constant.Common] + } + } + if content == "" { + content = title + } + return +} + +func (o *OfflinePushConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { + title, content, opts, err := o.getOfflinePushInfos(msg) + if err != nil { + return err + } + err = o.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) + if err != nil { + prommetrics.MsgOfflinePushFailedCounter.Inc() + return err + } + return nil +} diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index a61399fb6b..9521a84a07 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -19,20 +19,20 @@ type OnlinePusher interface { pushToUserIDs *[]string) []string } -type emptyOnlinePUsher struct{} +type emptyOnlinePusher struct{} -func newEmptyOnlinePUsher() *emptyOnlinePUsher { - return &emptyOnlinePUsher{} +func newEmptyOnlinePusher() *emptyOnlinePusher { + return &emptyOnlinePusher{} } -func (emptyOnlinePUsher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, +func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - log.ZWarn(ctx, "emptyOnlinePUsher GetConnsAndOnlinePush", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) return nil, nil } -func (u emptyOnlinePUsher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, +func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { - log.ZWarn(ctx, "emptyOnlinePUsher GetOnlinePushFailedUserIDs", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) return nil } @@ -45,7 +45,7 @@ func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) Onli case "etcd": return NewDefaultAllNode(disCov, config) default: - return newEmptyOnlinePUsher() + return newEmptyOnlinePusher() } } diff --git a/internal/push/push.go b/internal/push/push.go index 1a04bbea26..850f91d22e 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,6 +2,7 @@ package push import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -17,12 +18,12 @@ type pushServer struct { disCov discovery.SvcDiscoveryRegistry offlinePusher offlinepush.OfflinePusher pushCh *ConsumerHandler + offlinePushCh *OfflinePushConsumerHandler } type Config struct { RpcConfig config.Push RedisConfig config.Redis - MongodbConfig config.Mongo KafkaConfig config.Kafka NotificationConfig config.Notification Share config.Share @@ -55,18 +56,30 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - database := controller.NewPushDatabase(cacheModel) - consumer, err := NewConsumerHandler(config, offlinePusher, rdb, client) + database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig) + + consumer, err := NewConsumerHandler(config, database, offlinePusher, rdb, client) + if err != nil { + return err + } + + offlinePushConsumer, err := NewOfflinePushConsumerHandler(config, offlinePusher) if err != nil { return err } + pbpush.RegisterPushMsgServiceServer(server, &pushServer{ database: database, disCov: client, offlinePusher: offlinePusher, pushCh: consumer, + offlinePushCh: offlinePushConsumer, }) + go consumer.pushConsumerGroup.RegisterHandleAndConsumer(ctx, consumer) + + go offlinePushConsumer.OfflinePushConsumerGroup.RegisterHandleAndConsumer(ctx, offlinePushConsumer) + return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 03c299b7ab..4ecf20de52 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -1,33 +1,21 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package push import ( "context" "encoding/json" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" - pbchat "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -39,12 +27,17 @@ import ( "github.com/openimsdk/tools/utils/timeutil" "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" + "math/rand" + "strconv" + "time" ) type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + pushDatabase controller.PushDatabase + onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache msgRpcClient rpcclient.MessageRpcClient @@ -54,7 +47,7 @@ type ConsumerHandler struct { config *Config } -func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, +func NewConsumerHandler(config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) { var consumerHandler ConsumerHandler var err error @@ -63,51 +56,57 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, - &config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config + consumerHandler.pushDatabase = database + consumerHandler.onlineCache, err = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) + if err != nil { + return nil, err + } return &consumerHandler, nil } func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { - msgFromMQ := pbchat.PushMsgDataToMQ{} + msgFromMQ := pbpush.PushMsgReq{} if err := proto.Unmarshal(msg, &msgFromMQ); err != nil { log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg)) return } - pbData := &pbpush.PushMsgReq{ - MsgData: msgFromMQ.MsgData, - ConversationID: msgFromMQ.ConversationID, - } + sec := msgFromMQ.MsgData.SendTime / 1000 nowSec := timeutil.GetCurrentTimestampBySecond() - log.ZDebug(ctx, "push msg", "msg", pbData.String(), "sec", sec, "nowSec", nowSec) + if nowSec-sec > 10 { - return + prommetrics.MsgLoneTimePushCounter.Inc() + log.ZWarn(ctx, "it’s been a while since the message was sent", nil, "msg", msgFromMQ.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) } var err error + switch msgFromMQ.MsgData.SessionType { case constant.ReadGroupChatType: - err = c.Push2Group(ctx, pbData.MsgData.GroupID, pbData.MsgData) + err = c.Push2Group(ctx, msgFromMQ.MsgData.GroupID, msgFromMQ.MsgData) default: var pushUserIDList []string - isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) - if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) + isSenderSync := datautil.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) + if !isSenderSync || msgFromMQ.MsgData.SendID == msgFromMQ.MsgData.RecvID { + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID) } else { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID, msgFromMQ.MsgData.SendID) } - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) + err = c.Push2User(ctx, pushUserIDList, msgFromMQ.MsgData) } if err != nil { - log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) + log.ZWarn(ctx, "push failed", err, "msg", msgFromMQ.String()) } } @@ -116,6 +115,14 @@ func (*ConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } func (*ConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + c.onlineCache.Lock.Lock() + for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver { + c.onlineCache.Cond.Wait() + } + c.onlineCache.Lock.Unlock() + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + log.ZInfo(ctx, "begin consume messages") + for msg := range claim.Messages() { ctx := c.pushConsumerGroup.GetContextFromMsg(msg) c.handleMs2PsChat(ctx, msg.Value) @@ -125,21 +132,28 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. -func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) error { - log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) +func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "msg", msg.String(), "time cost", t) + }(time.Now()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, userIDs) + log.ZInfo(ctx, "webhookBeforeOnlinePush end") + + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } - log.ZDebug(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) + log.ZInfo(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) if !c.shouldPushOffline(ctx, msg) { return nil } + log.ZInfo(ctx, "shouldPushOffline end") for _, v := range wsResults { //message sender do not need offline push @@ -151,17 +165,17 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * return nil } } - offlinePUshUserID := []string{msg.RecvID} + offlinePushUserID := []string{msg.RecvID} //receiver offline push if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, - offlinePUshUserID, msg, nil); err != nil { + offlinePushUserID, msg, nil); err != nil { return err } - - err = c.offlinePushMsg(ctx, msg, offlinePUshUserID) + log.ZInfo(ctx, "webhookBeforeOfflinePush end") + err = c.offlinePushMsg(ctx, msg, offlinePushUserID) if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePUshUserID", offlinePUshUserID, "msg", msg) + log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) return nil } @@ -179,58 +193,94 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat return true } +func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { + onlineUserIDs, offlineUserIDs, err := c.onlineCache.GetUsersOnline(ctx, pushToUserIDs) + if err != nil { + return nil, err + } + + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) + var result []*msggateway.SingleMsgToUserResults + if len(onlineUserIDs) > 0 { + var err error + result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, onlineUserIDs) + if err != nil { + return nil, err + } + } + for _, userID := range offlineUserIDs { + result = append(result, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return result, nil +} + func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get super group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg end", "msg", msg.String(), "groupID", groupID, "time cost", t) + }(time.Now()) var pushToUserIDs []string if err = c.webhookBeforeGroupOnlinePush(ctx, &c.config.WebhooksConfig.BeforeGroupOnlinePush, groupID, msg, &pushToUserIDs); err != nil { return err } + log.ZInfo(ctx, "webhookBeforeGroupOnlinePush end") err = c.groupMessagesHandler(ctx, groupID, &pushToUserIDs, msg) if err != nil { return err } + log.ZInfo(ctx, "groupMessagesHandler end") - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } - log.ZDebug(ctx, "group push result", "result", wsResults, "msg", msg) + log.ZInfo(ctx, "group push result", "result", wsResults, "msg", msg) if !c.shouldPushOffline(ctx, msg) { return nil } needOfflinePushUserIDs := c.onlinePusher.GetOnlinePushFailedUserIDs(ctx, msg, wsResults, &pushToUserIDs) - + log.ZInfo(ctx, "GetOnlinePushFailedUserIDs end") //filter some user, like don not disturb or don't need offline push etc. needOfflinePushUserIDs, err = c.filterGroupMessageOfflinePush(ctx, groupID, msg, needOfflinePushUserIDs) if err != nil { return err } + log.ZInfo(ctx, "filterGroupMessageOfflinePush end") + // Use offline push messaging if len(needOfflinePushUserIDs) > 0 { - var offlinePushUserIDs []string - err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) - if err != nil { - return err - } - - if len(offlinePushUserIDs) > 0 { - needOfflinePushUserIDs = offlinePushUserIDs - } + c.asyncOfflinePush(ctx, needOfflinePushUserIDs, msg) + } - err = c.offlinePushMsg(ctx, msg, needOfflinePushUserIDs) - if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg) - return nil - } + return nil +} +func (c *ConsumerHandler) asyncOfflinePush(ctx context.Context, needOfflinePushUserIDs []string, msg *sdkws.MsgData) { + var offlinePushUserIDs []string + err := c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) + if err != nil { + log.ZWarn(ctx, "webhookBeforeOfflinePush failed", err, "msg", msg) + return } - return nil + if len(offlinePushUserIDs) > 0 { + needOfflinePushUserIDs = offlinePushUserIDs + } + if err := c.pushDatabase.MsgToOfflinePushMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(msg.SendID, msg.RecvID), needOfflinePushUserIDs, msg); err != nil { + log.ZError(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs", + needOfflinePushUserIDs, "msg", msg) + prommetrics.SingleChatMsgProcessFailedCounter.Inc() + return + } } + func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) @@ -264,7 +314,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri if unmarshalNotificationElem(msg.Content, &tips) != nil { return err } - log.ZInfo(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) + log.ZDebug(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) if len(c.config.Share.IMAdminUserID) > 0 { ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) } @@ -348,6 +398,7 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten } return } + func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) maxSeq, err := c.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) @@ -356,6 +407,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, } return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } + func unmarshalNotificationElem(bytes []byte, t any) error { var notification sdkws.NotificationElem if err := json.Unmarshal(bytes, ¬ification); err != nil { diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index df9267ae04..6f77164e38 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -16,13 +16,16 @@ package conversation import ( "context" + "sort" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/db/redisutil" - "sort" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -40,10 +43,11 @@ import ( ) type conversationServer struct { - msgRpcClient *rpcclient.MessageRpcClient - user *rpcclient.UserRpcClient - groupRpcClient *rpcclient.GroupRpcClient - conversationDatabase controller.ConversationDatabase + msgRpcClient *rpcclient.MessageRpcClient + user *rpcclient.UserRpcClient + groupRpcClient *rpcclient.GroupRpcClient + conversationDatabase controller.ConversationDatabase + conversationNotificationSender *ConversationNotificationSender config *Config } @@ -184,21 +188,31 @@ func (c *conversationServer) GetAllConversations(ctx context.Context, req *pbcon } func (c *conversationServer) GetConversations(ctx context.Context, req *pbconversation.GetConversationsReq) (*pbconversation.GetConversationsResp, error) { - conversations, err := c.conversationDatabase.FindConversations(ctx, req.OwnerUserID, req.ConversationIDs) + conversations, err := c.getConversations(ctx, req.OwnerUserID, req.ConversationIDs) + if err != nil { + return nil, err + } + return &pbconversation.GetConversationsResp{ + Conversations: conversations, + }, nil +} + +func (c *conversationServer) getConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { + conversations, err := c.conversationDatabase.FindConversations(ctx, ownerUserID, conversationIDs) if err != nil { return nil, err } resp := &pbconversation.GetConversationsResp{Conversations: []*pbconversation.Conversation{}} resp.Conversations = convert.ConversationsDB2Pb(conversations) - return resp, nil + return convert.ConversationsDB2Pb(conversations), nil } func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { - var conversation tablerelation.Conversation + var conversation dbModel.Conversation if err := datautil.CopyStructFields(&conversation, req.Conversation); err != nil { return nil, err } - err := c.conversationDatabase.SetUserConversations(ctx, req.Conversation.OwnerUserID, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.SetUserConversations(ctx, req.Conversation.OwnerUserID, []*dbModel.Conversation{&conversation}) if err != nil { return nil, err } @@ -207,11 +221,11 @@ func (c *conversationServer) SetConversation(ctx context.Context, req *pbconvers return resp, nil } -// nolint func (c *conversationServer) SetConversations(ctx context.Context, req *pbconversation.SetConversationsReq) (*pbconversation.SetConversationsResp, error) { if req.Conversation == nil { return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") } + if req.Conversation.ConversationType == constant.WriteGroupChatType { groupInfo, err := c.groupRpcClient.GetGroupInfo(ctx, req.Conversation.GroupID) if err != nil { @@ -221,98 +235,141 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed") } } - var unequal int - var conv tablerelation.Conversation - if len(req.UserIDs) == 1 { - cs, err := c.conversationDatabase.FindConversations(ctx, req.UserIDs[0], []string{req.Conversation.ConversationID}) + + conversationMap := make(map[string]*dbModel.Conversation) + var needUpdateUsersList []string + + for _, userID := range req.UserIDs { + conversationList, err := c.conversationDatabase.FindConversations(ctx, userID, []string{req.Conversation.ConversationID}) if err != nil { return nil, err } - if len(cs) == 0 { - return nil, errs.ErrRecordNotFound.WrapMsg("conversation not found") + if len(conversationList) != 0 { + conversationMap[userID] = conversationList[0] + } else { + needUpdateUsersList = append(needUpdateUsersList, userID) } - conv = *cs[0] } - var conversation tablerelation.Conversation + + var conversation dbModel.Conversation conversation.ConversationID = req.Conversation.ConversationID conversation.ConversationType = req.Conversation.ConversationType conversation.UserID = req.Conversation.UserID conversation.GroupID = req.Conversation.GroupID + m := make(map[string]any) - if req.Conversation.RecvMsgOpt != nil { - m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value - if req.Conversation.RecvMsgOpt.Value != conv.RecvMsgOpt { - unequal++ + + setConversationFieldsFunc := func() { + if req.Conversation.RecvMsgOpt != nil { + m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value } - } - if req.Conversation.AttachedInfo != nil { - m["attached_info"] = req.Conversation.AttachedInfo.Value - if req.Conversation.AttachedInfo.Value != conv.AttachedInfo { - unequal++ + if req.Conversation.AttachedInfo != nil { + m["attached_info"] = req.Conversation.AttachedInfo.Value } - } - if req.Conversation.Ex != nil { - m["ex"] = req.Conversation.Ex.Value - if req.Conversation.Ex.Value != conv.Ex { - unequal++ + if req.Conversation.Ex != nil { + m["ex"] = req.Conversation.Ex.Value } - } - if req.Conversation.IsPinned != nil { - m["is_pinned"] = req.Conversation.IsPinned.Value - if req.Conversation.IsPinned.Value != conv.IsPinned { - unequal++ + if req.Conversation.IsPinned != nil { + m["is_pinned"] = req.Conversation.IsPinned.Value } - } - if req.Conversation.GroupAtType != nil { - m["group_at_type"] = req.Conversation.GroupAtType.Value - if req.Conversation.GroupAtType.Value != conv.GroupAtType { - unequal++ + if req.Conversation.GroupAtType != nil { + m["group_at_type"] = req.Conversation.GroupAtType.Value } - } - if req.Conversation.MsgDestructTime != nil { - m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value - if req.Conversation.MsgDestructTime.Value != conv.MsgDestructTime { - unequal++ + if req.Conversation.MsgDestructTime != nil { + m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + } + if req.Conversation.MsgDestructTime != nil { + m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + } + if req.Conversation.BurnDuration != nil { + m["burn_duration"] = req.Conversation.BurnDuration.Value } } - if req.Conversation.IsMsgDestruct != nil { - m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value - if req.Conversation.IsMsgDestruct.Value != conv.IsMsgDestruct { - unequal++ + + // set need set field in conversation + setConversationFieldsFunc() + + for userID := range conversationMap { + unequal := len(m) + + if req.Conversation.RecvMsgOpt != nil { + if req.Conversation.RecvMsgOpt.Value == conversationMap[userID].RecvMsgOpt { + unequal-- + } + } + + if req.Conversation.AttachedInfo != nil { + if req.Conversation.AttachedInfo.Value == conversationMap[userID].AttachedInfo { + unequal-- + } + } + + if req.Conversation.Ex != nil { + if req.Conversation.Ex.Value == conversationMap[userID].Ex { + unequal-- + } + } + if req.Conversation.IsPinned != nil { + if req.Conversation.IsPinned.Value == conversationMap[userID].IsPinned { + unequal-- + } + } + + if req.Conversation.GroupAtType != nil { + if req.Conversation.GroupAtType.Value == conversationMap[userID].GroupAtType { + unequal-- + } + } + + if req.Conversation.MsgDestructTime != nil { + if req.Conversation.MsgDestructTime.Value == conversationMap[userID].MsgDestructTime { + unequal-- + } + } + + if req.Conversation.IsMsgDestruct != nil { + if req.Conversation.IsMsgDestruct.Value == conversationMap[userID].IsMsgDestruct { + unequal-- + } + } + + if req.Conversation.BurnDuration != nil { + if req.Conversation.BurnDuration.Value == conversationMap[userID].BurnDuration { + unequal-- + } + } + + if unequal > 0 { + needUpdateUsersList = append(needUpdateUsersList, userID) } } + if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { - var conversations []*tablerelation.Conversation + var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { - conversation2 := conversation - conversation2.OwnerUserID = ownerUserID - conversation2.IsPrivateChat = req.Conversation.IsPrivateChat.Value - conversations = append(conversations, &conversation2) + transConversation := conversation + transConversation.OwnerUserID = ownerUserID + transConversation.IsPrivateChat = req.Conversation.IsPrivateChat.Value + conversations = append(conversations, &transConversation) } if err := c.conversationDatabase.SyncPeerUserPrivateConversationTx(ctx, conversations); err != nil { return nil, err } + for _, userID := range req.UserIDs { c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } - } - - if req.Conversation.BurnDuration != nil { - m["burn_duration"] = req.Conversation.BurnDuration.Value - if req.Conversation.BurnDuration.Value != conv.BurnDuration { - unequal++ - } - } - - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, req.UserIDs, &conversation, m); err != nil { - return nil, err - } + } else { + if len(m) != 0 && len(needUpdateUsersList) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } - if unequal > 0 { - for _, v := range req.UserIDs { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } } } @@ -330,12 +387,12 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, ) (*pbconversation.CreateSingleChatConversationsResp, error) { switch req.ConversationType { case constant.SingleChatType: - var conversation tablerelation.Conversation + var conversation dbModel.Conversation conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.SendID conversation.UserID = req.RecvID - err := c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation", conversation) } @@ -343,17 +400,17 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, conversation2 := conversation conversation2.OwnerUserID = req.RecvID conversation2.UserID = req.SendID - err = c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation2}) + err = c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation2}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } case constant.NotificationChatType: - var conversation tablerelation.Conversation + var conversation dbModel.Conversation conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.RecvID conversation.UserID = req.SendID - err := c.conversationDatabase.CreateConversation(ctx, []*tablerelation.Conversation{&conversation}) + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } @@ -378,6 +435,14 @@ func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbc return &pbconversation.SetConversationMaxSeqResp{}, nil } +func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, + map[string]any{"min_seq": req.MinSeq}); err != nil { + return nil, err + } + return &pbconversation.SetConversationMinSeqResp{}, nil +} + func (c *conversationServer) GetConversationIDs(ctx context.Context, req *pbconversation.GetConversationIDsReq) (*pbconversation.GetConversationIDsResp, error) { conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) if err != nil { @@ -574,6 +639,9 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv if req.MaxSeq != nil { m["max_seq"] = req.MaxSeq.Value } + if req.LatestMsgDestructTime != nil { + m["latest_msg_destruct_time"] = time.UnixMilli(req.LatestMsgDestructTime.Value) + } if len(m) > 0 { if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.UserIDs, req.ConversationID, m); err != nil { return nil, err @@ -581,3 +649,64 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv } return &pbconversation.UpdateConversationResp{}, nil } + +func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbconversation.GetOwnerConversationReq) (*pbconversation.GetOwnerConversationResp, error) { + total, conversations, err := c.conversationDatabase.GetOwnerConversation(ctx, req.UserID, req.Pagination) + if err != nil { + return nil, err + } + return &pbconversation.GetOwnerConversationResp{ + Total: total, + Conversations: convert.ConversationsDB2Pb(conversations), + }, nil +} + +func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Context, _ *pbconversation.GetConversationsNeedDestructMsgsReq) (*pbconversation.GetConversationsNeedDestructMsgsResp, error) { + num, err := c.conversationDatabase.GetAllConversationIDsNumber(ctx) + if err != nil { + log.ZError(ctx, "GetAllConversationIDsNumber failed", err) + return nil, err + } + const batchNum = 100 + + if num == 0 { + return nil, errs.New("Need Destruct Msg is nil").Wrap() + } + + maxPage := (num + batchNum - 1) / batchNum + + temp := make([]*model.Conversation, 0, maxPage*batchNum) + + for pageNumber := 0; pageNumber < int(maxPage); pageNumber++ { + pagination := &sdkws.RequestPagination{ + PageNumber: int32(pageNumber), + ShowNumber: batchNum, + } + + conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) + if err != nil { + // log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) + continue + } + + // log.ZDebug(ctx, "PageConversationIDs success", "pageNumber", pageNumber, "conversationIDsNum", len(conversationIDs), "conversationIDs", conversationIDs) + if len(conversationIDs) == 0 { + continue + } + + conversations, err := c.conversationDatabase.GetConversationsByConversationID(ctx, conversationIDs) + if err != nil { + log.ZError(ctx, "GetConversationsByConversationID failed", err, "conversationIDs", conversationIDs) + continue + } + + for _, conversation := range conversations { + if conversation.IsMsgDestruct && conversation.MsgDestructTime != 0 && ((time.Now().UnixMilli() > (conversation.MsgDestructTime + conversation.LatestMsgDestructTime.UnixMilli() + 8*60*60)) || // 8*60*60 is UTC+8 + conversation.LatestMsgDestructTime.IsZero()) { + temp = append(temp, conversation) + } + } + } + + return &pbconversation.GetConversationsNeedDestructMsgsResp{Conversations: convert.ConversationsDB2Pb(temp)}, nil +} diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go new file mode 100644 index 0000000000..ad88b2bbd5 --- /dev/null +++ b/internal/rpc/conversation/sync.go @@ -0,0 +1,56 @@ +package conversation + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/conversation" +) + +func (c *conversationServer) GetFullOwnerConversationIDs(ctx context.Context, req *conversation.GetFullOwnerConversationIDsReq) (*conversation.GetFullOwnerConversationIDsResp, error) { + vl, err := c.conversationDatabase.FindMaxConversationUserVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(conversationIDs) + if req.IdHash == idHash { + conversationIDs = nil + } + return &conversation.GetFullOwnerConversationIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + ConversationIDs: conversationIDs, + }, nil +} + +func (c *conversationServer) GetIncrementalConversation(ctx context.Context, req *conversation.GetIncrementalConversationReq) (*conversation.GetIncrementalConversationResp, error) { + opt := incrversion.Option[*conversation.Conversation, conversation.GetIncrementalConversationResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: c.conversationDatabase.FindConversationUserVersion, + CacheMaxVersion: c.conversationDatabase.FindMaxConversationUserVersionCache, + Find: func(ctx context.Context, conversationIDs []string) ([]*conversation.Conversation, error) { + return c.getConversations(ctx, req.UserID, conversationIDs) + }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*conversation.Conversation, full bool) *conversation.GetIncrementalConversationResp { + return &conversation.GetIncrementalConversationResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index f31c4587c5..5e3dc9b9c3 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -16,6 +16,8 @@ package group import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" - "time" ) // CallbackBeforeCreateGroup callback before create group. @@ -100,27 +101,45 @@ func (s *groupServer) webhookAfterCreateGroup(ctx context.Context, after *config s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateGroupResp{}, after) } -func (s *groupServer) webhookBeforeMemberJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMember *model.GroupMember, groupEx string) error { +func (s *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMembers []*model.GroupMember, groupID string, groupEx string) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - cbReq := &callbackstruct.CallbackBeforeMemberJoinGroupReq{ - CallbackCommand: callbackstruct.CallbackBeforeMemberJoinGroupCommand, - GroupID: groupMember.GroupID, - UserID: groupMember.UserID, - Ex: groupMember.Ex, + groupMembersMap := datautil.SliceToMap(groupMembers, func(e *model.GroupMember) string { + return e.UserID + }) + var groupMembersCallback []*callbackstruct.CallbackGroupMember + + for _, member := range groupMembers { + groupMembersCallback = append(groupMembersCallback, &callbackstruct.CallbackGroupMember{ + UserID: member.UserID, + Ex: member.Ex, + }) + } + + cbReq := &callbackstruct.CallbackBeforeMembersJoinGroupReq{ + CallbackCommand: callbackstruct.CallbackBeforeMembersJoinGroupCommand, + GroupID: groupID, + MembersList: groupMembersCallback, GroupEx: groupEx, } - resp := &callbackstruct.CallbackBeforeMemberJoinGroupResp{} + resp := &callbackstruct.CallbackBeforeMembersJoinGroupResp{} + if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } - if resp.MuteEndTime != nil { - groupMember.MuteEndTime = time.UnixMilli(*resp.MuteEndTime) + for _, memberCallbackResp := range resp.MemberCallbackList { + if _, ok := groupMembersMap[(*memberCallbackResp.UserID)]; ok { + if memberCallbackResp.MuteEndTime != nil { + groupMembersMap[(*memberCallbackResp.UserID)].MuteEndTime = time.UnixMilli(*memberCallbackResp.MuteEndTime) + } + + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].FaceURL, memberCallbackResp.FaceURL) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].Ex, memberCallbackResp.Ex) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].Nickname, memberCallbackResp.Nickname) + datautil.NotNilReplace(&groupMembersMap[(*memberCallbackResp.UserID)].RoleLevel, memberCallbackResp.RoleLevel) + } } - datautil.NotNilReplace(&groupMember.FaceURL, resp.FaceURL) - datautil.NotNilReplace(&groupMember.Ex, resp.Ex) - datautil.NotNilReplace(&groupMember.Nickname, resp.Nickname) - datautil.NotNilReplace(&groupMember.RoleLevel, resp.RoleLevel) + return nil }) } @@ -244,10 +263,13 @@ func (s *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before return err } - if len(resp.RefusedMembersAccount) > 0 { - // Handle the scenario where certain members are refused - // You might want to update the req.Members list or handle it as per your business logic - } + // Handle the scenario where certain members are refused + // You might want to update the req.Members list or handle it as per your business logic + + // if len(resp.RefusedMembersAccount) > 0 { + // implement members are refused + // } + return nil }) } @@ -336,3 +358,74 @@ func (s *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *confi } s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoResp{}, after) } + +func (s *groupServer) webhookBeforeSetGroupInfoEX(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoEXReq) error { + return webhook.WithCondition(ctx, before, func(ctx context.Context) error { + cbReq := &callbackstruct.CallbackBeforeSetGroupInfoEXReq{ + CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoCommand, + GroupID: req.GroupInfoForSetEX.GroupID, + GroupName: req.GroupInfoForSetEX.GroupName, + Notification: req.GroupInfoForSetEX.Notification, + Introduction: req.GroupInfoForSetEX.Introduction, + FaceURL: req.GroupInfoForSetEX.FaceURL, + } + + if req.GroupInfoForSetEX.Ex != nil { + cbReq.Ex = req.GroupInfoForSetEX.Ex + } + log.ZDebug(ctx, "debug CallbackBeforeSetGroupInfoEX", "ex", cbReq.Ex) + + if req.GroupInfoForSetEX.NeedVerification != nil { + cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + } + if req.GroupInfoForSetEX.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + } + if req.GroupInfoForSetEX.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + } + + resp := &callbackstruct.CallbackBeforeSetGroupInfoEXResp{} + + if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + return err + } + + datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupID, &resp.GroupID) + datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupName, &resp.GroupName) + datautil.NotNilReplace(&req.GroupInfoForSetEX.FaceURL, &resp.FaceURL) + datautil.NotNilReplace(&req.GroupInfoForSetEX.Introduction, &resp.Introduction) + datautil.NotNilReplace(&req.GroupInfoForSetEX.Ex, &resp.Ex) + datautil.NotNilReplace(&req.GroupInfoForSetEX.NeedVerification, &resp.NeedVerification) + datautil.NotNilReplace(&req.GroupInfoForSetEX.LookMemberInfo, &resp.LookMemberInfo) + datautil.NotNilReplace(&req.GroupInfoForSetEX.ApplyMemberFriend, &resp.ApplyMemberFriend) + + return nil + }) +} + +func (s *groupServer) webhookAfterSetGroupInfoEX(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoEXReq) { + cbReq := &callbackstruct.CallbackAfterSetGroupInfoEXReq{ + CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoCommand, + GroupID: req.GroupInfoForSetEX.GroupID, + GroupName: req.GroupInfoForSetEX.GroupName, + Notification: req.GroupInfoForSetEX.Notification, + Introduction: req.GroupInfoForSetEX.Introduction, + FaceURL: req.GroupInfoForSetEX.FaceURL, + } + + if req.GroupInfoForSetEX.Ex != nil { + cbReq.Ex = req.GroupInfoForSetEX.Ex + } + if req.GroupInfoForSetEX.NeedVerification != nil { + cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + } + if req.GroupInfoForSetEX.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + } + if req.GroupInfoForSetEX.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + } + + s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoEXResp{}, after) +} diff --git a/internal/rpc/group/convert.go b/internal/rpc/group/convert.go index a75693904d..8026430c30 100644 --- a/internal/rpc/group/convert.go +++ b/internal/rpc/group/convert.go @@ -57,3 +57,7 @@ func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel InviterUserID: member.InviterUserID, } } + +func (s *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { + return s.groupMemberDB2PB(member, 0) +} diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index b4b503b950..08895f9c5f 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -54,6 +54,39 @@ func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[s return m } +func UpdateGroupInfoEXMap(ctx context.Context, group *sdkws.GroupInfoForSetEX) map[string]any { + m := make(map[string]any) + + if group.GroupName != "" { + m["group_name"] = group.GroupName + } + if group.Notification != nil { + m["notification"] = group.Notification.Value + m["notification_update_time"] = time.Now() + m["notification_user_id"] = mcontext.GetOpUserID(ctx) + } + if group.Introduction != nil { + m["introduction"] = group.Introduction.Value + } + if group.FaceURL != nil { + m["face_url"] = group.FaceURL.Value + } + if group.NeedVerification != nil { + m["need_verification"] = group.NeedVerification.Value + } + if group.LookMemberInfo != nil { + m["look_member_info"] = group.LookMemberInfo.Value + } + if group.ApplyMemberFriend != nil { + m["apply_member_friend"] = group.ApplyMemberFriend.Value + } + if group.Ex != nil { + m["ex"] = group.Ex.Value + } + + return m +} + func UpdateGroupStatusMap(status int) map[string]any { return map[string]any{ "status": status, diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index a9cea4ff22..c45a7827b7 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -17,17 +17,18 @@ package group import ( "context" "fmt" + "math/big" + "math/rand" + "strconv" + "strings" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "math/big" - "math/rand" - "strconv" - "strings" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" @@ -104,13 +105,20 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database gs.user = userRpcClient - gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { - users, err := userRpcClient.GetUsersInfo(ctx, userIDs) - if err != nil { - return nil, err - } - return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil - }) + gs.notification = NewGroupNotificationSender( + database, + &msgRpcClient, + &userRpcClient, + &conversationRpcClient, + config, + func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { + users, err := userRpcClient.GetUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil + }, + ) localcache.InitLocalCache(&config.LocalCacheConfig) gs.conversationRpcClient = conversationRpcClient gs.msgRpcClient = msgRpcClient @@ -120,8 +128,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return nil } -func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgroup.NotificationUserInfoUpdateReq) (*pbgroup.NotificationUserInfoUpdateResp, error) { - members, err := s.db.FindGroupMemberUser(ctx, nil, req.UserID) +func (g *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgroup.NotificationUserInfoUpdateReq) (*pbgroup.NotificationUserInfoUpdateResp, error) { + members, err := g.db.FindGroupMemberUser(ctx, nil, req.UserID) if err != nil { return nil, err } @@ -133,18 +141,22 @@ func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro groupIDs = append(groupIDs, member.GroupID) } for _, groupID := range groupIDs { - s.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) + if err := g.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } + for _, groupID := range groupIDs { + g.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) } - if err = s.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { + if err = g.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { return nil, err } - return &pbgroup.NotificationUserInfoUpdateResp{}, nil } -func (s *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - groupMember, err := s.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) +func (g *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + groupMember, err := g.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) if err != nil { return err } @@ -155,11 +167,11 @@ func (s *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error return nil } -func (s *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string, complete bool) (map[string]*sdkws.PublicUserInfo, error) { +func (g *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string, complete bool) (map[string]*sdkws.PublicUserInfo, error) { if len(userIDs) == 0 { return map[string]*sdkws.PublicUserInfo{}, nil } - users, err := s.user.GetPublicUserInfos(ctx, userIDs, complete) + users, err := g.user.GetPublicUserInfos(ctx, userIDs, complete) if err != nil { return nil, err } @@ -168,16 +180,16 @@ func (s *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string }), nil } -func (s *groupServer) IsNotFound(err error) bool { +func (g *groupServer) IsNotFound(err error) bool { return errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) } -func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { +func (g *groupServer) GenGroupID(ctx context.Context, groupID *string) error { if *groupID != "" { - _, err := s.db.TakeGroup(ctx, *groupID) + _, err := g.db.TakeGroup(ctx, *groupID) if err == nil { return servererrs.ErrGroupIDExisted.WrapMsg("group id existed " + *groupID) - } else if s.IsNotFound(err) { + } else if g.IsNotFound(err) { return nil } else { return err @@ -188,10 +200,10 @@ func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { bi := big.NewInt(0) bi.SetString(id[0:8], 16) id = bi.String() - _, err := s.db.TakeGroup(ctx, id) + _, err := g.db.TakeGroup(ctx, id) if err == nil { continue - } else if s.IsNotFound(err) { + } else if g.IsNotFound(err) { *groupID = id return nil } else { @@ -201,14 +213,14 @@ func (s *groupServer) GenGroupID(ctx context.Context, groupID *string) error { return servererrs.ErrData.WrapMsg("group id gen error") } -func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupReq) (*pbgroup.CreateGroupResp, error) { +func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupReq) (*pbgroup.CreateGroupResp, error) { if req.GroupInfo.GroupType != constant.WorkingGroup { return nil, errs.ErrArgs.WrapMsg(fmt.Sprintf("group type only supports %d", constant.WorkingGroup)) } if req.OwnerUserID == "" { return nil, errs.ErrArgs.WrapMsg("no group owner") } - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -222,7 +234,7 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, errs.ErrArgs.WrapMsg("group member repeated") } - userMap, err := s.user.GetUsersInfoMap(ctx, userIDs) + userMap, err := g.user.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -231,17 +243,17 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, servererrs.ErrUserIDNotFound.WrapMsg("user not found") } - if err := s.webhookBeforeCreateGroup(ctx, &s.config.WebhooksConfig.BeforeCreateGroup, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeCreateGroup(ctx, &g.config.WebhooksConfig.BeforeCreateGroup, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } var groupMembers []*model.GroupMember group := convert.Pb2DBGroupInfo(req.GroupInfo) - if err := s.GenGroupID(ctx, &group.GroupID); err != nil { + if err := g.GenGroupID(ctx, &group.GroupID); err != nil { return nil, err } - joinGroup := func(userID string, roleLevel int32) error { + joinGroupFunc := func(userID string, roleLevel int32) { groupMember := &model.GroupMember{ GroupID: group.GroupID, UserID: userID, @@ -253,26 +265,24 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { - return err - } groupMembers = append(groupMembers, groupMember) - return nil - } - if err := joinGroup(req.OwnerUserID, constant.GroupOwner); err != nil { - return nil, err } + + joinGroupFunc(req.OwnerUserID, constant.GroupOwner) + for _, userID := range req.AdminUserIDs { - if err := joinGroup(userID, constant.GroupAdmin); err != nil { - return nil, err - } + joinGroupFunc(userID, constant.GroupAdmin) } + for _, userID := range req.MemberUserIDs { - if err := joinGroup(userID, constant.GroupOrdinaryUsers); err != nil { - return nil, err - } + joinGroupFunc(userID, constant.GroupOrdinaryUsers) + } + + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err } - if err := s.db.CreateGroup(ctx, []*model.Group{group}, groupMembers); err != nil { + + if err := g.db.CreateGroup(ctx, []*model.Group{group}, groupMembers); err != nil { return nil, err } resp := &pbgroup.CreateGroupResp{GroupInfo: &sdkws.GroupInfo{}} @@ -282,17 +292,17 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR tips := &sdkws.GroupCreatedTips{ Group: resp.GroupInfo, OperationTime: group.CreateTime.UnixMilli(), - GroupOwnerUser: s.groupMemberDB2PB(groupMembers[0], userMap[groupMembers[0].UserID].AppMangerLevel), + GroupOwnerUser: g.groupMemberDB2PB(groupMembers[0], userMap[groupMembers[0].UserID].AppMangerLevel), } for _, member := range groupMembers { member.Nickname = userMap[member.UserID].Nickname - tips.MemberList = append(tips.MemberList, s.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel)) + tips.MemberList = append(tips.MemberList, g.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel)) if member.UserID == opUserID { - tips.OpUser = s.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel) + tips.OpUser = g.groupMemberDB2PB(member, userMap[member.UserID].AppMangerLevel) break } } - s.notification.GroupCreatedNotification(ctx, tips) + g.notification.GroupCreatedNotification(ctx, tips) reqCallBackAfter := &pbgroup.CreateGroupReq{ MemberUserIDs: userIDs, @@ -301,16 +311,16 @@ func (s *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR AdminUserIDs: req.AdminUserIDs, } - s.webhookAfterCreateGroup(ctx, &s.config.WebhooksConfig.AfterCreateGroup, reqCallBackAfter) + g.webhookAfterCreateGroup(ctx, &g.config.WebhooksConfig.AfterCreateGroup, reqCallBackAfter) return resp, nil } -func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJoinedGroupListReq) (*pbgroup.GetJoinedGroupListResp, error) { - if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { +func (g *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJoinedGroupListReq) (*pbgroup.GetJoinedGroupListResp, error) { + if err := authverify.CheckAccessV3(ctx, req.FromUserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } - total, members, err := s.db.PageGetJoinGroup(ctx, req.FromUserID, req.Pagination) + total, members, err := g.db.PageGetJoinGroup(ctx, req.FromUserID, req.Pagination) if err != nil { return nil, err } @@ -322,19 +332,19 @@ func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJo groupIDs := datautil.Slice(members, func(e *model.GroupMember) string { return e.GroupID }) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { @@ -352,14 +362,14 @@ func (s *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJo return &resp, nil } -func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.InviteUserToGroupReq) (*pbgroup.InviteUserToGroupResp, error) { +func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.InviteUserToGroupReq) (*pbgroup.InviteUserToGroupResp, error) { if len(req.InvitedUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("user empty") } if datautil.Duplicate(req.InvitedUserIDs) { return nil, errs.ErrArgs.WrapMsg("userID duplicate") } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -368,7 +378,7 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed checking group status found it dismissed") } - userMap, err := s.user.GetUsersInfoMap(ctx, req.InvitedUserIDs) + userMap, err := g.user.GetUsersInfoMap(ctx, req.InvitedUserIDs) if err != nil { return nil, err } @@ -379,24 +389,24 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite var groupMember *model.GroupMember var opUserID string - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { opUserID = mcontext.GetOpUserID(ctx) var err error - groupMember, err = s.db.TakeGroupMember(ctx, req.GroupID, opUserID) + groupMember, err = g.db.TakeGroupMember(ctx, req.GroupID, opUserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, groupMember); err != nil { + if err := g.PopulateGroupMember(ctx, groupMember); err != nil { return nil, err } } - if err := s.webhookBeforeInviteUserToGroup(ctx, &s.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeInviteUserToGroup(ctx, &g.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } if group.NeedVerification == constant.AllNeedVerification { - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if !(groupMember.RoleLevel == constant.GroupOwner || groupMember.RoleLevel == constant.GroupAdmin) { var requests []*model.GroupRequest for _, userID := range req.InvitedUserIDs { @@ -409,11 +419,11 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite HandledTime: time.Unix(0, 0), }) } - if err := s.db.CreateGroupRequest(ctx, requests); err != nil { + if err := g.db.CreateGroupRequest(ctx, requests); err != nil { return nil, err } for _, request := range requests { - s.notification.JoinGroupApplicationNotification(ctx, &pbgroup.JoinGroupReq{ + g.notification.JoinGroupApplicationNotification(ctx, &pbgroup.JoinGroupReq{ GroupID: request.GroupID, ReqMessage: request.ReqMsg, JoinSource: request.JoinSource, @@ -437,28 +447,29 @@ func (s *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { - return nil, err - } groupMembers = append(groupMembers, member) + } + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, groupMembers, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err } - if err := s.db.CreateGroup(ctx, nil, groupMembers); err != nil { + + if err := g.db.CreateGroup(ctx, nil, groupMembers); err != nil { return nil, err } - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, req.InvitedUserIDs); err != nil { + + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.InvitedUserIDs...); err != nil { return nil, err } - s.notification.MemberInvitedNotification(ctx, req.GroupID, req.Reason, req.InvitedUserIDs) return &pbgroup.InviteUserToGroupResp{}, nil } -func (s *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGroupAllMemberReq) (*pbgroup.GetGroupAllMemberResp, error) { - members, err := s.db.FindGroupMemberAll(ctx, req.GroupID) +func (g *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGroupAllMemberReq) (*pbgroup.GetGroupAllMemberResp, error) { + members, err := g.db.FindGroupMemberAll(ctx, req.GroupID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } var resp pbgroup.GetGroupAllMemberResp @@ -468,21 +479,21 @@ func (s *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGro return &resp, nil } -func (s *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { +func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { var ( total int64 members []*model.GroupMember err error ) if req.Keyword == "" { - total, members, err = s.db.PageGetGroupMember(ctx, req.GroupID, req.Pagination) + total, members, err = g.db.PageGetGroupMember(ctx, req.GroupID, req.Pagination) } else { - members, err = s.db.FindGroupMemberAll(ctx, req.GroupID) + members, err = g.db.FindGroupMemberAll(ctx, req.GroupID) } if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } if req.Keyword != "" { @@ -512,8 +523,8 @@ func (s *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGr }, nil } -func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGroupMemberReq) (*pbgroup.KickGroupMemberResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGroupMemberReq) (*pbgroup.KickGroupMemberResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -527,18 +538,26 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou if datautil.Contain(opUserID, req.KickedUserIDs...) { return nil, errs.ErrArgs.WrapMsg("opUserID in KickedUserIDs") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, append(req.KickedUserIDs, opUserID)) + owner, err := g.db.TakeGroupOwner(ctx, req.GroupID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if datautil.Contain(owner.UserID, req.KickedUserIDs...) { + return nil, errs.ErrArgs.WrapMsg("ownerUID can not Kick") + } + + members, err := g.db.FindGroupMembers(ctx, req.GroupID, append(req.KickedUserIDs, opUserID)) + if err != nil { + return nil, err + } + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } memberMap := make(map[string]*model.GroupMember) for i, member := range members { memberMap[member.UserID] = members[i] } - isAppManagerUid := authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) opMember := memberMap[opUserID] for _, userID := range req.KickedUserIDs { member, ok := memberMap[userID] @@ -562,11 +581,11 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou } } } - num, err := s.db.FindGroupMemberNum(ctx, req.GroupID) + num, err := g.db.FindGroupMemberNum(ctx, req.GroupID) if err != nil { return nil, err } - ownerUserIDs, err := s.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupOwner) + ownerUserIDs, err := g.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupOwner) if err != nil { return nil, err } @@ -574,7 +593,7 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou if len(ownerUserIDs) > 0 { ownerUserID = ownerUserIDs[0] } - if err := s.db.DeleteGroupMember(ctx, group.GroupID, req.KickedUserIDs); err != nil { + if err := g.db.DeleteGroupMember(ctx, group.GroupID, req.KickedUserIDs); err != nil { return nil, err } tips := &sdkws.MemberKickedTips{ @@ -586,7 +605,7 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou FaceURL: group.FaceURL, OwnerUserID: ownerUserID, CreateTime: group.CreateTime.UnixMilli(), - MemberCount: num, + MemberCount: num - uint32(len(req.KickedUserIDs)), Ex: group.Ex, Status: group.Status, CreatorUserID: group.CreatorUserID, @@ -605,39 +624,50 @@ func (s *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou for _, userID := range req.KickedUserIDs { tips.KickedUserList = append(tips.KickedUserList, convert.Db2PbGroupMember(memberMap[userID])) } - s.notification.MemberKickedNotification(ctx, tips) - if err := s.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { + g.notification.MemberKickedNotification(ctx, tips) + if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { return nil, err } - s.webhookAfterKickGroupMember(ctx, &s.config.WebhooksConfig.AfterKickGroupMember, req) + g.webhookAfterKickGroupMember(ctx, &g.config.WebhooksConfig.AfterKickGroupMember, req) return &pbgroup.KickGroupMemberResp{}, nil } -func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetGroupMembersInfoReq) (*pbgroup.GetGroupMembersInfoResp, error) { +func (g *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetGroupMembersInfoReq) (*pbgroup.GetGroupMembersInfoResp, error) { if len(req.UserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDs empty") } if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, req.UserIDs) + members, err := g.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { - return nil, err - } return &pbgroup.GetGroupMembersInfoResp{ - Members: datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { - return convert.Db2PbGroupMember(e) - }), + Members: members, }, nil } +func (g *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + members, err := g.db.FindGroupMembers(ctx, groupID, userIDs) + if err != nil { + return nil, err + } + if err := g.PopulateGroupMember(ctx, members...); err != nil { + return nil, err + } + return datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { + return convert.Db2PbGroupMember(e) + }), nil +} + // GetGroupApplicationList handles functions that get a list of group requests. -func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { - groupIDs, err := s.db.FindUserManagedGroupID(ctx, req.FromUserID) +func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { + groupIDs, err := g.db.FindUserManagedGroupID(ctx, req.FromUserID) if err != nil { return nil, err } @@ -645,7 +675,7 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. if len(groupIDs) == 0 { return resp, nil } - total, groupRequests, err := s.db.PageGroupRequest(ctx, groupIDs, req.Pagination) + total, groupRequests, err := g.db.PageGroupRequest(ctx, groupIDs, req.Pagination) if err != nil { return nil, err } @@ -659,11 +689,11 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. userIDs = append(userIDs, gr.UserID) } userIDs = datautil.Distinct(userIDs) - userMap, err := s.user.GetPublicUserInfoMap(ctx, userIDs, true) + userMap, err := g.user.GetPublicUserInfoMap(ctx, userIDs, true) if err != nil { return nil, err } - groups, err := s.db.FindGroup(ctx, datautil.Distinct(groupIDs)) + groups, err := g.db.FindGroup(ctx, datautil.Distinct(groupIDs)) if err != nil { return nil, err } @@ -673,15 +703,15 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. if ids := datautil.Single(datautil.Keys(groupMap), groupIDs); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { @@ -697,45 +727,56 @@ func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. return resp, nil } -func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsInfoReq) (*pbgroup.GetGroupsInfoResp, error) { +func (g *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsInfoReq) (*pbgroup.GetGroupsInfoResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupID is empty") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := g.getGroupsInfo(ctx, req.GroupIDs) if err != nil { return nil, err } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, req.GroupIDs) + return &pbgroup.GetGroupsInfoResp{ + GroupInfos: groups, + }, nil +} + +func (g *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { + if len(groupIDs) == 0 { + return nil, nil + } + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, req.GroupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) + if err != nil { + return nil, err + } + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - return &pbgroup.GetGroupsInfoResp{ - GroupInfos: datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { - var ownerUserID string - if owner, ok := ownerMap[e.GroupID]; ok { - ownerUserID = owner.UserID - } - return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) - }), - }, nil + return datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { + var ownerUserID string + if owner, ok := ownerMap[e.GroupID]; ok { + ownerUserID = owner.UserID + } + return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) + }), nil } -func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { +func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { if !datautil.Contain(req.HandleResult, constant.GroupResponseAgree, constant.GroupResponseRefuse) { return nil, errs.ErrArgs.WrapMsg("HandleResult unknown") } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - groupMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + groupMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -743,11 +784,11 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") } } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } - groupRequest, err := s.db.TakeGroupRequest(ctx, req.GroupID, req.FromUserID) + groupRequest, err := g.db.TakeGroupRequest(ctx, req.GroupID, req.FromUserID) if err != nil { return nil, err } @@ -755,12 +796,12 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup return nil, servererrs.ErrGroupRequestHandled.WrapMsg("group request already processed") } var inGroup bool - if _, err := s.db.TakeGroupMember(ctx, req.GroupID, req.FromUserID); err == nil { + if _, err := g.db.TakeGroupMember(ctx, req.GroupID, req.FromUserID); err == nil { inGroup = true // Already in group - } else if !s.IsNotFound(err) { + } else if !g.IsNotFound(err) { return nil, err } - if _, err := s.user.GetPublicUserInfo(ctx, req.FromUserID); err != nil { + if _, err := g.user.GetPublicUserInfo(ctx, req.FromUserID); err != nil { return nil, err } var member *model.GroupMember @@ -776,40 +817,39 @@ func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup MuteEndTime: time.Unix(0, 0), InviterUserID: groupRequest.InviterUserID, OperatorUserID: mcontext.GetOpUserID(ctx), - Ex: groupRequest.Ex, } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, member, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{member}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } } log.ZDebug(ctx, "GroupApplicationResponse", "inGroup", inGroup, "HandleResult", req.HandleResult, "member", member) - if err := s.db.HandlerGroupRequest(ctx, req.GroupID, req.FromUserID, req.HandledMsg, req.HandleResult, member); err != nil { + if err := g.db.HandlerGroupRequest(ctx, req.GroupID, req.FromUserID, req.HandledMsg, req.HandleResult, member); err != nil { return nil, err } switch req.HandleResult { case constant.GroupResponseAgree: - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, []string{req.FromUserID}); err != nil { - return nil, err - } - s.notification.GroupApplicationAcceptedNotification(ctx, req) + g.notification.GroupApplicationAcceptedNotification(ctx, req) if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - s.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID) + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, groupRequest.InviterUserID, req.FromUserID); err != nil { + return nil, err + } } case constant.GroupResponseRefuse: - s.notification.GroupApplicationRejectedNotification(ctx, req) + g.notification.GroupApplicationRejectedNotification(ctx, req) } return &pbgroup.GroupApplicationResponseResp{}, nil } -func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { - user, err := s.user.GetUserInfo(ctx, req.InviterUserID) +func (g *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { + user, err := g.user.GetUserInfo(ctx, req.InviterUserID) if err != nil { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -825,14 +865,14 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) Ex: req.Ex, } - if err := s.webhookBeforeApplyJoinGroup(ctx, &s.config.WebhooksConfig.BeforeApplyJoinGroup, reqCall); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeApplyJoinGroup(ctx, &g.config.WebhooksConfig.BeforeApplyJoinGroup, reqCall); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - _, err = s.db.TakeGroupMember(ctx, req.GroupID, req.InviterUserID) + _, err = g.db.TakeGroupMember(ctx, req.GroupID, req.InviterUserID) if err == nil { return nil, errs.ErrArgs.Wrap() - } else if !s.IsNotFound(err) && errs.Unwrap(err) != errs.ErrRecordNotFound { + } else if !g.IsNotFound(err) && errs.Unwrap(err) != errs.ErrRecordNotFound { return nil, err } log.ZDebug(ctx, "JoinGroup.groupInfo", "group", group, "eq", group.NeedVerification == constant.Directly) @@ -847,22 +887,22 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) MuteEndTime: time.UnixMilli(0), } - if err := s.webhookBeforeMemberJoinGroup(ctx, &s.config.WebhooksConfig.BeforeMemberJoinGroup, groupMember, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeMembersJoinGroup(ctx, &g.config.WebhooksConfig.BeforeMemberJoinGroup, []*model.GroupMember{groupMember}, group.GroupID, group.Ex); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if err := s.db.CreateGroup(ctx, nil, []*model.GroupMember{groupMember}); err != nil { + if err := g.db.CreateGroup(ctx, nil, []*model.GroupMember{groupMember}); err != nil { return nil, err } - if err := s.conversationRpcClient.GroupChatFirstCreateConversation(ctx, req.GroupID, []string{req.InviterUserID}); err != nil { + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID); err != nil { return nil, err } - s.notification.MemberEnterNotification(ctx, req.GroupID, req.InviterUserID) - s.webhookAfterJoinGroup(ctx, &s.config.WebhooksConfig.AfterJoinGroup, req) + g.webhookAfterJoinGroup(ctx, &g.config.WebhooksConfig.AfterJoinGroup, req) return &pbgroup.JoinGroupResp{}, nil } + groupRequest := model.GroupRequest{ UserID: req.InviterUserID, ReqMsg: req.ReqMessage, @@ -872,74 +912,74 @@ func (s *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) HandledTime: time.Unix(0, 0), Ex: req.Ex, } - if err = s.db.CreateGroupRequest(ctx, []*model.GroupRequest{&groupRequest}); err != nil { + if err = g.db.CreateGroupRequest(ctx, []*model.GroupRequest{&groupRequest}); err != nil { return nil, err } - s.notification.JoinGroupApplicationNotification(ctx, req) + g.notification.JoinGroupApplicationNotification(ctx, req) return &pbgroup.JoinGroupResp{}, nil } -func (s *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) (*pbgroup.QuitGroupResp, error) { +func (g *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) (*pbgroup.QuitGroupResp, error) { if req.UserID == "" { req.UserID = mcontext.GetOpUserID(ctx) } else { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } } - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } if member.RoleLevel == constant.GroupOwner { return nil, errs.ErrNoPermission.WrapMsg("group owner can't quit") } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - err = s.db.DeleteGroupMember(ctx, req.GroupID, []string{req.UserID}) + err = g.db.DeleteGroupMember(ctx, req.GroupID, []string{req.UserID}) if err != nil { return nil, err } - s.notification.MemberQuitNotification(ctx, s.groupMemberDB2PB(member, 0)) - if err := s.deleteMemberAndSetConversationSeq(ctx, req.GroupID, []string{req.UserID}); err != nil { + g.notification.MemberQuitNotification(ctx, g.groupMemberDB2PB(member, 0)) + if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, []string{req.UserID}); err != nil { return nil, err } - s.webhookAfterQuitGroup(ctx, &s.config.WebhooksConfig.AfterQuitGroup, req) + g.webhookAfterQuitGroup(ctx, &g.config.WebhooksConfig.AfterQuitGroup, req) return &pbgroup.QuitGroupResp{}, nil } -func (s *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { +func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conevrsationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := s.msgRpcClient.GetConversationMaxSeq(ctx, conevrsationID) + maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conevrsationID) if err != nil { return err } - return s.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) + return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) } -func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { +func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { var opMember *model.GroupMember - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { var err error - opMember, err = s.db.TakeGroupMember(ctx, req.GroupInfoForSet.GroupID, mcontext.GetOpUserID(ctx)) + opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSet.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } if !(opMember.RoleLevel == constant.GroupOwner || opMember.RoleLevel == constant.GroupAdmin) { return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") } - if err := s.PopulateGroupMember(ctx, opMember); err != nil { + if err := g.PopulateGroupMember(ctx, opMember); err != nil { return nil, err } } - if err := s.webhookBeforeSetGroupInfo(ctx, &s.config.WebhooksConfig.BeforeSetGroupInfo, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeSetGroupInfo(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfo, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) if err != nil { return nil, err } @@ -947,35 +987,35 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return nil, servererrs.ErrDismissedAlready.Wrap() } - count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + count, err := g.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err } - owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + owner, err := g.db.TakeGroupOwner(ctx, group.GroupID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owner); err != nil { + if err := g.PopulateGroupMember(ctx, owner); err != nil { return nil, err } update := UpdateGroupInfoMap(ctx, req.GroupInfoForSet) if len(update) == 0 { return &pbgroup.SetGroupInfoResp{}, nil } - if err := s.db.UpdateGroup(ctx, group.GroupID, update); err != nil { + if err := g.db.UpdateGroup(ctx, group.GroupID, update); err != nil { return nil, err } - group, err = s.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) + group, err = g.db.TakeGroup(ctx, req.GroupInfoForSet.GroupID) if err != nil { return nil, err } tips := &sdkws.GroupInfoSetTips{ - Group: s.groupDB2PB(group, owner.UserID, count), + Group: g.groupDB2PB(group, owner.UserID, count), MuteTime: 0, OpUser: &sdkws.GroupMemberFullInfo{}, } if opMember != nil { - tips.OpUser = s.groupMemberDB2PB(opMember, 0) + tips.OpUser = g.groupMemberDB2PB(opMember, 0) } num := len(update) if req.GroupInfoForSet.Notification != "" { @@ -986,33 +1026,146 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf ConversationType: constant.ReadGroupChatType, GroupID: req.GroupInfoForSet.GroupID, } - resp, err := s.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := s.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() - s.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) } if req.GroupInfoForSet.GroupName != "" { num-- - s.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) } if num > 0 { - s.notification.GroupInfoSetNotification(ctx, tips) + g.notification.GroupInfoSetNotification(ctx, tips) } - s.webhookAfterSetGroupInfo(ctx, &s.config.WebhooksConfig.AfterSetGroupInfo, req) + g.webhookAfterSetGroupInfo(ctx, &g.config.WebhooksConfig.AfterSetGroupInfo, req) return &pbgroup.SetGroupInfoResp{}, nil } -func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) (*pbgroup.TransferGroupOwnerResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupInfoEXReq) (*pbgroup.SetGroupInfoEXResp, error) { + var opMember *model.GroupMember + + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + var err error + + opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSetEX.GroupID, mcontext.GetOpUserID(ctx)) + if err != nil { + return nil, err + } + + if !(opMember.RoleLevel == constant.GroupOwner || opMember.RoleLevel == constant.GroupAdmin) { + return nil, errs.ErrNoPermission.WrapMsg("no group owner or admin") + } + + if err := g.PopulateGroupMember(ctx, opMember); err != nil { + return nil, err + } + } + + if err := g.webhookBeforeSetGroupInfoEX(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfoEX, req); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + + group, err := g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } + + count, err := g.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + + owner, err := g.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + + if err := g.PopulateGroupMember(ctx, owner); err != nil { + return nil, err + } + + updatedData := UpdateGroupInfoEXMap(ctx, req.GroupInfoForSetEX) + if len(updatedData) == 0 { + return &pbgroup.SetGroupInfoEXResp{}, nil + } + + if err := g.db.UpdateGroup(ctx, group.GroupID, updatedData); err != nil { + return nil, err + } + + group, err = g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + if err != nil { + return nil, err + } + + tips := &sdkws.GroupInfoSetTips{ + Group: g.groupDB2PB(group, owner.UserID, count), + MuteTime: 0, + OpUser: &sdkws.GroupMemberFullInfo{}, + } + + if opMember != nil { + tips.OpUser = g.groupMemberDB2PB(opMember, 0) + } + + num := len(updatedData) + if req.GroupInfoForSetEX.Notification != nil { + num-- + + if req.GroupInfoForSetEX.Notification.Value != "" { + func() { + conversation := &pbconversation.ConversationReq{ + ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSetEX.GroupID), + ConversationType: constant.ReadGroupChatType, + GroupID: req.GroupInfoForSetEX.GroupID, + } + + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSetEX.GroupID}) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) + return + } + + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} + + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) + } + }() + + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + } + } + + if req.GroupInfoForSetEX.GroupName != "" { + num-- + g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) + } + + if num > 0 { + g.notification.GroupInfoSetNotification(ctx, tips) + } + + g.webhookAfterSetGroupInfoEX(ctx, &g.config.WebhooksConfig.AfterSetGroupInfoEX, req) + + return &pbgroup.SetGroupInfoEXResp{}, nil +} + +func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) (*pbgroup.TransferGroupOwnerResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -1022,11 +1175,11 @@ func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans if req.OldOwnerUserID == req.NewOwnerUserID { return nil, errs.ErrArgs.WrapMsg("OldOwnerUserID == NewOwnerUserID") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, []string{req.OldOwnerUserID, req.NewOwnerUserID}) + members, err := g.db.FindGroupMembers(ctx, req.GroupID, []string{req.OldOwnerUserID, req.NewOwnerUserID}) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } memberMap := datautil.SliceToMap(members, func(e *model.GroupMember) string { return e.UserID }) @@ -1041,33 +1194,33 @@ func (s *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans if newOwner == nil { return nil, errs.ErrArgs.WrapMsg("NewOwnerUser not in group " + req.NewOwnerUserID) } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if !(mcontext.GetOpUserID(ctx) == oldOwner.UserID && oldOwner.RoleLevel == constant.GroupOwner) { return nil, errs.ErrNoPermission.WrapMsg("no permission transfer group owner") } } - if err := s.db.TransferGroupOwner(ctx, req.GroupID, req.OldOwnerUserID, req.NewOwnerUserID, newOwner.RoleLevel); err != nil { + if err := g.db.TransferGroupOwner(ctx, req.GroupID, req.OldOwnerUserID, req.NewOwnerUserID, newOwner.RoleLevel); err != nil { return nil, err } - s.webhookAfterTransferGroupOwner(ctx, &s.config.WebhooksConfig.AfterTransferGroupOwner, req) + g.webhookAfterTransferGroupOwner(ctx, &g.config.WebhooksConfig.AfterTransferGroupOwner, req) - s.notification.GroupOwnerTransferredNotification(ctx, req) + g.notification.GroupOwnerTransferredNotification(ctx, req) return &pbgroup.TransferGroupOwnerResp{}, nil } -func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) (*pbgroup.GetGroupsResp, error) { +func (g *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) (*pbgroup.GetGroupsResp, error) { var ( group []*model.Group err error ) var resp pbgroup.GetGroupsResp if req.GroupID != "" { - group, err = s.db.FindGroup(ctx, []string{req.GroupID}) + group, err = g.db.FindGroup(ctx, []string{req.GroupID}) resp.Total = uint32(len(group)) } else { var total int64 - total, group, err = s.db.SearchGroup(ctx, req.GroupName, req.Pagination) + total, group, err = g.db.SearchGroup(ctx, req.GroupName, req.Pagination) resp.Total = uint32(total) } @@ -1079,7 +1232,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) return e.GroupID }) - ownerMembers, err := s.db.FindGroupsOwner(ctx, groupIDs) + ownerMembers, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } @@ -1087,7 +1240,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) ownerMemberMap := datautil.SliceToMap(ownerMembers, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNumMap, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } @@ -1105,14 +1258,14 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) return &resp, nil } -func (s *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGroupMembersCMSReq) (*pbgroup.GetGroupMembersCMSResp, error) { - total, members, err := s.db.SearchGroupMember(ctx, req.UserName, req.GroupID, req.Pagination) +func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGroupMembersCMSReq) (*pbgroup.GetGroupMembersCMSResp, error) { + total, members, err := g.db.SearchGroupMember(ctx, req.UserName, req.GroupID, req.Pagination) if err != nil { return nil, err } var resp pbgroup.GetGroupMembersCMSResp resp.Total = uint32(total) - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } resp.Members = datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { @@ -1121,12 +1274,12 @@ func (s *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGr return &resp, nil } -func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { - user, err := s.user.GetPublicUserInfo(ctx, req.UserID) +func (g *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { + user, err := g.user.GetPublicUserInfo(ctx, req.UserID) if err != nil { return nil, err } - total, requests, err := s.db.PageGroupRequestUser(ctx, req.UserID, req.Pagination) + total, requests, err := g.db.PageGroupRequestUser(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -1136,24 +1289,24 @@ func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgrou groupIDs := datautil.Distinct(datautil.Slice(requests, func(e *model.GroupRequest) string { return e.GroupID })) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } groupMap := datautil.SliceToMap(groups, func(e *model.Group) string { return e.GroupID }) - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } @@ -1169,44 +1322,44 @@ func (s *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgrou }, nil } -func (s *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGroupReq) (*pbgroup.DismissGroupResp, error) { - owner, err := s.db.TakeGroupOwner(ctx, req.GroupID) +func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGroupReq) (*pbgroup.DismissGroupResp, error) { + owner, err := g.db.TakeGroupOwner(ctx, req.GroupID) if err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if owner.UserID != mcontext.GetOpUserID(ctx) { return nil, errs.ErrNoPermission.WrapMsg("not group owner") } } - if err := s.PopulateGroupMember(ctx, owner); err != nil { + if err := g.PopulateGroupMember(ctx, owner); err != nil { return nil, err } - group, err := s.db.TakeGroup(ctx, req.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } if !req.DeleteMember && group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.WrapMsg("group status is dismissed") } - if err := s.db.DismissGroup(ctx, req.GroupID, req.DeleteMember); err != nil { + if err := g.db.DismissGroup(ctx, req.GroupID, req.DeleteMember); err != nil { return nil, err } if !req.DeleteMember { - num, err := s.db.FindGroupMemberNum(ctx, req.GroupID) + num, err := g.db.FindGroupMemberNum(ctx, req.GroupID) if err != nil { return nil, err } tips := &sdkws.GroupDismissedTips{ - Group: s.groupDB2PB(group, owner.UserID, num), + Group: g.groupDB2PB(group, owner.UserID, num), OpUser: &sdkws.GroupMemberFullInfo{}, } if mcontext.GetOpUserID(ctx) == owner.UserID { - tips.OpUser = s.groupMemberDB2PB(owner, 0) + tips.OpUser = g.groupMemberDB2PB(owner, 0) } - s.notification.GroupDismissedNotification(ctx, tips) + g.notification.GroupDismissedNotification(ctx, tips) } - membersID, err := s.db.FindGroupMemberUserID(ctx, group.GroupID) + membersID, err := g.db.FindGroupMemberUserID(ctx, group.GroupID) if err != nil { return nil, err } @@ -1217,21 +1370,21 @@ func (s *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou GroupType: string(group.GroupType), } - s.webhookAfterDismissGroup(ctx, &s.config.WebhooksConfig.AfterDismissGroup, cbReq) + g.webhookAfterDismissGroup(ctx, &g.config.WebhooksConfig.AfterDismissGroup, cbReq) return &pbgroup.DismissGroupResp{}, nil } -func (s *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGroupMemberReq) (*pbgroup.MuteGroupMemberResp, error) { - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) +func (g *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGroupMemberReq) (*pbgroup.MuteGroupMemberResp, error) { + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - opMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -1249,23 +1402,23 @@ func (s *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGrou } } data := UpdateGroupMemberMutedTimeMap(time.Now().Add(time.Second * time.Duration(req.MutedSeconds))) - if err := s.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { + if err := g.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { return nil, err } - s.notification.GroupMemberMutedNotification(ctx, req.GroupID, req.UserID, req.MutedSeconds) + g.notification.GroupMemberMutedNotification(ctx, req.GroupID, req.UserID, req.MutedSeconds) return &pbgroup.MuteGroupMemberResp{}, nil } -func (s *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.CancelMuteGroupMemberReq) (*pbgroup.CancelMuteGroupMemberResp, error) { - member, err := s.db.TakeGroupMember(ctx, req.GroupID, req.UserID) +func (g *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.CancelMuteGroupMemberReq) (*pbgroup.CancelMuteGroupMemberResp, error) { + member, err := g.db.TakeGroupMember(ctx, req.GroupID, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, member); err != nil { + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { - opMember, err := s.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -1283,36 +1436,36 @@ func (s *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.Ca } } data := UpdateGroupMemberMutedTimeMap(time.Unix(0, 0)) - if err := s.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { + if err := g.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { return nil, err } - s.notification.GroupMemberCancelMutedNotification(ctx, req.GroupID, req.UserID) + g.notification.GroupMemberCancelMutedNotification(ctx, req.GroupID, req.UserID) return &pbgroup.CancelMuteGroupMemberResp{}, nil } -func (s *groupServer) MuteGroup(ctx context.Context, req *pbgroup.MuteGroupReq) (*pbgroup.MuteGroupResp, error) { - if err := s.CheckGroupAdmin(ctx, req.GroupID); err != nil { +func (g *groupServer) MuteGroup(ctx context.Context, req *pbgroup.MuteGroupReq) (*pbgroup.MuteGroupResp, error) { + if err := g.CheckGroupAdmin(ctx, req.GroupID); err != nil { return nil, err } - if err := s.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupStatusMuted)); err != nil { + if err := g.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupStatusMuted)); err != nil { return nil, err } - s.notification.GroupMutedNotification(ctx, req.GroupID) + g.notification.GroupMutedNotification(ctx, req.GroupID) return &pbgroup.MuteGroupResp{}, nil } -func (s *groupServer) CancelMuteGroup(ctx context.Context, req *pbgroup.CancelMuteGroupReq) (*pbgroup.CancelMuteGroupResp, error) { - if err := s.CheckGroupAdmin(ctx, req.GroupID); err != nil { +func (g *groupServer) CancelMuteGroup(ctx context.Context, req *pbgroup.CancelMuteGroupReq) (*pbgroup.CancelMuteGroupResp, error) { + if err := g.CheckGroupAdmin(ctx, req.GroupID); err != nil { return nil, err } - if err := s.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupOk)); err != nil { + if err := g.db.UpdateGroup(ctx, req.GroupID, UpdateGroupStatusMap(constant.GroupOk)); err != nil { return nil, err } - s.notification.GroupCancelMutedNotification(ctx, req.GroupID) + g.notification.GroupCancelMutedNotification(ctx, req.GroupID) return &pbgroup.CancelMuteGroupResp{}, nil } -func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGroupMemberInfoReq) (*pbgroup.SetGroupMemberInfoResp, error) { +func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGroupMemberInfoReq) (*pbgroup.SetGroupMemberInfoResp, error) { if len(req.Members) == 0 { return nil, errs.ErrArgs.WrapMsg("members empty") } @@ -1320,7 +1473,7 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if opUserID == "" { return nil, errs.ErrNoPermission.WrapMsg("no op user id") } - isAppManagerUid := authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) for i := range req.Members { req.Members[i].FaceURL = nil } @@ -1350,7 +1503,7 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if _, ok := temp[opUserID]; !ok { userIDs = append(userIDs, opUserID) } - dbMembers, err := s.db.FindGroupMembers(ctx, groupID, userIDs) + dbMembers, err := g.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } @@ -1405,12 +1558,12 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr for i := 0; i < len(req.Members); i++ { - if err := s.webhookBeforeSetGroupMemberInfo(ctx, &s.config.WebhooksConfig.BeforeSetGroupMemberInfo, req.Members[i]); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeSetGroupMemberInfo(ctx, &g.config.WebhooksConfig.BeforeSetGroupMemberInfo, req.Members[i]); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } } - if err := s.db.UpdateGroupMembers(ctx, datautil.Slice(req.Members, func(e *pbgroup.SetGroupMemberInfo) *common.BatchUpdateGroupMember { + if err := g.db.UpdateGroupMembers(ctx, datautil.Slice(req.Members, func(e *pbgroup.SetGroupMemberInfo) *common.BatchUpdateGroupMember { return &common.BatchUpdateGroupMember{ GroupID: e.GroupID, UserID: e.UserID, @@ -1423,30 +1576,30 @@ func (s *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if member.RoleLevel != nil { switch member.RoleLevel.Value { case constant.GroupAdmin: - s.notification.GroupMemberSetToAdminNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberSetToAdminNotification(ctx, member.GroupID, member.UserID) case constant.GroupOrdinaryUsers: - s.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) } } if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil { - s.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) + g.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) } } for i := 0; i < len(req.Members); i++ { - s.webhookAfterSetGroupMemberInfo(ctx, &s.config.WebhooksConfig.AfterSetGroupMemberInfo, req.Members[i]) + g.webhookAfterSetGroupMemberInfo(ctx, &g.config.WebhooksConfig.AfterSetGroupMemberInfo, req.Members[i]) } return &pbgroup.SetGroupMemberInfoResp{}, nil } -func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.GetGroupAbstractInfoReq) (*pbgroup.GetGroupAbstractInfoResp, error) { +func (g *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.GetGroupAbstractInfoReq) (*pbgroup.GetGroupAbstractInfoResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupIDs empty") } if datautil.Duplicate(req.GroupIDs) { return nil, errs.ErrArgs.WrapMsg("groupIDs duplicate") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := g.db.FindGroup(ctx, req.GroupIDs) if err != nil { return nil, err } @@ -1455,7 +1608,7 @@ func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.Get })); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg("not found group " + strings.Join(ids, ",")) } - groupUserMap, err := s.db.MapGroupMemberUserID(ctx, req.GroupIDs) + groupUserMap, err := g.db.MapGroupMemberUserID(ctx, req.GroupIDs) if err != nil { return nil, err } @@ -1470,15 +1623,15 @@ func (s *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.Get }, nil } -func (s *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.GetUserInGroupMembersReq) (*pbgroup.GetUserInGroupMembersResp, error) { +func (g *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.GetUserInGroupMembersReq) (*pbgroup.GetUserInGroupMembersResp, error) { if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupIDs empty") } - members, err := s.db.FindGroupMemberUser(ctx, req.GroupIDs, req.UserID) + members, err := g.db.FindGroupMemberUser(ctx, req.GroupIDs, req.UserID) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } return &pbgroup.GetUserInGroupMembersResp{ @@ -1488,8 +1641,8 @@ func (s *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.Ge }, nil } -func (s *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetGroupMemberUserIDsReq) (*pbgroup.GetGroupMemberUserIDsResp, error) { - userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) +func (g *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetGroupMemberUserIDsReq) (*pbgroup.GetGroupMemberUserIDsResp, error) { + userIDs, err := g.db.FindGroupMemberUserID(ctx, req.GroupID) if err != nil { return nil, err } @@ -1498,15 +1651,15 @@ func (s *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.Ge }, nil } -func (s *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup.GetGroupMemberRoleLevelReq) (*pbgroup.GetGroupMemberRoleLevelResp, error) { +func (g *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup.GetGroupMemberRoleLevelReq) (*pbgroup.GetGroupMemberRoleLevelResp, error) { if len(req.RoleLevels) == 0 { return nil, errs.ErrArgs.WrapMsg("RoleLevels empty") } - members, err := s.db.FindGroupMemberRoleLevels(ctx, req.GroupID, req.RoleLevels) + members, err := g.db.FindGroupMemberRoleLevels(ctx, req.GroupID, req.RoleLevels) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, members...); err != nil { + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } return &pbgroup.GetGroupMemberRoleLevelResp{ @@ -1516,8 +1669,8 @@ func (s *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup. }, nil } -func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req *pbgroup.GetGroupUsersReqApplicationListReq) (*pbgroup.GetGroupUsersReqApplicationListResp, error) { - requests, err := s.db.FindGroupRequests(ctx, req.GroupID, req.UserIDs) +func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req *pbgroup.GetGroupUsersReqApplicationListReq) (*pbgroup.GetGroupUsersReqApplicationListResp, error) { + requests, err := g.db.FindGroupRequests(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err } @@ -1527,7 +1680,7 @@ func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * groupIDs := datautil.Distinct(datautil.Slice(requests, func(e *model.GroupRequest) string { return e.GroupID })) - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } @@ -1537,17 +1690,17 @@ func (s *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * if ids := datautil.Single(groupIDs, datautil.Keys(groupMap)); len(ids) > 0 { return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - owners, err := s.db.FindGroupsOwner(ctx, groupIDs) + owners, err := g.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } - if err := s.PopulateGroupMember(ctx, owners...); err != nil { + if err := g.PopulateGroupMember(ctx, owners...); err != nil { return nil, err } ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - groupMemberNum, err := s.db.MapGroupMemberNum(ctx, groupIDs) + groupMemberNum, err := g.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index cfa62c85db..64e922fe2b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -16,30 +16,52 @@ package group import ( "context" + "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" + "go.mongodb.org/mongo-driver/mongo" ) -func NewGroupNotificationSender(db controller.GroupDatabase, msgRpcClient *rpcclient.MessageRpcClient, userRpcClient *rpcclient.UserRpcClient, config *Config, fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error)) *GroupNotificationSender { +// GroupApplicationReceiver +const ( + applicantReceiver = iota + adminReceiver +) + +func NewGroupNotificationSender( + db controller.GroupDatabase, + msgRpcClient *rpcclient.MessageRpcClient, + userRpcClient *rpcclient.UserRpcClient, + conversationRpcClient *rpcclient.ConversationRpcClient, + config *Config, + fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error), +) *GroupNotificationSender { return &GroupNotificationSender{ NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(msgRpcClient), rpcclient.WithUserRpcClient(userRpcClient)), getUsersInfo: fn, db: db, config: config, + + conversationRpcClient: conversationRpcClient, + msgRpcClient: msgRpcClient, } } @@ -48,6 +70,9 @@ type GroupNotificationSender struct { getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) db controller.GroupDatabase config *Config + + conversationRpcClient *rpcclient.ConversationRpcClient + msgRpcClient *rpcclient.MessageRpcClient } func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, members ...*model.GroupMember) error { @@ -118,25 +143,8 @@ func (g *GroupNotificationSender) getGroupInfo(ctx context.Context, groupID stri if len(ownerUserIDs) > 0 { ownerUserID = ownerUserIDs[0] } - return &sdkws.GroupInfo{ - GroupID: gm.GroupID, - GroupName: gm.GroupName, - Notification: gm.Notification, - Introduction: gm.Introduction, - FaceURL: gm.FaceURL, - OwnerUserID: ownerUserID, - CreateTime: gm.CreateTime.UnixMilli(), - MemberCount: num, - Ex: gm.Ex, - Status: gm.Status, - CreatorUserID: gm.CreatorUserID, - GroupType: gm.GroupType, - NeedVerification: gm.NeedVerification, - LookMemberInfo: gm.LookMemberInfo, - ApplyMemberFriend: gm.ApplyMemberFriend, - NotificationUpdateTime: gm.NotificationUpdateTime.UnixMilli(), - NotificationUserID: gm.NotificationUserID, - }, nil + + return convert.Db2PbGroupInfo(gm, ownerUserID, num), nil } func (g *GroupNotificationSender) getGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { @@ -190,29 +198,6 @@ func (g *GroupNotificationSender) getGroupOwnerAndAdminUserID(ctx context.Contex return datautil.Slice(members, fn), nil } -//nolint:unused -func (g *GroupNotificationSender) groupDB2PB(group *model.Group, ownerUserID string, memberCount uint32) *sdkws.GroupInfo { - return &sdkws.GroupInfo{ - GroupID: group.GroupID, - GroupName: group.GroupName, - Notification: group.Notification, - Introduction: group.Introduction, - FaceURL: group.FaceURL, - OwnerUserID: ownerUserID, - CreateTime: group.CreateTime.UnixMilli(), - MemberCount: memberCount, - Ex: group.Ex, - Status: group.Status, - CreatorUserID: group.CreatorUserID, - GroupType: group.GroupType, - NeedVerification: group.NeedVerification, - LookMemberInfo: group.LookMemberInfo, - ApplyMemberFriend: group.ApplyMemberFriend, - NotificationUpdateTime: group.NotificationUpdateTime.UnixMilli(), - NotificationUserID: group.NotificationUserID, - } -} - func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { return &sdkws.GroupMemberFullInfo{ GroupID: member.GroupID, @@ -243,10 +228,13 @@ func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, ap } */ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { + return g.fillOpUserByUserID(ctx, mcontext.GetOpUserID(ctx), opUser, groupID) +} + +func (g *GroupNotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { if opUser == nil { return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } - userID := mcontext.GetOpUserID(ctx) if groupID != "" { if authverify.IsManagerUserID(userID, g.config.Share.IMAdminUserID) { *opUser = &sdkws.GroupMemberFullInfo{ @@ -259,7 +247,7 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws member, err := g.db.TakeGroupMember(ctx, groupID, userID) if err == nil { *opUser = g.groupMemberDB2PB(member, 0) - } else if !errs.ErrRecordNotFound.Is(err) { + } else if !(errors.Is(err, mongo.ErrNoDocuments) || errs.ErrRecordNotFound.Is(err)) { return err } } @@ -287,6 +275,32 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + return + } + } +} + +func (g *GroupNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == model.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -297,6 +311,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -310,6 +325,7 @@ func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -323,6 +339,7 @@ func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } @@ -336,6 +353,7 @@ func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -380,6 +398,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -400,15 +419,17 @@ func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx conte if err != nil { return } - tips := &sdkws.GroupApplicationAcceptedTips{Group: group, HandleMsg: req.HandledMsg} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + + var opUser *sdkws.GroupMemberFullInfo + if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } for _, userID := range append(userIDs, req.FromUserID) { + tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { - tips.ReceiverAs = 0 + tips.ReceiverAs = applicantReceiver } else { - tips.ReceiverAs = 1 + tips.ReceiverAs = adminReceiver } g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationAcceptedNotification, tips) } @@ -431,15 +452,17 @@ func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx conte if err != nil { return } - tips := &sdkws.GroupApplicationRejectedTips{Group: group, HandleMsg: req.HandledMsg} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + + var opUser *sdkws.GroupMemberFullInfo + if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } for _, userID := range append(userIDs, req.FromUserID) { + tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { - tips.ReceiverAs = 0 + tips.ReceiverAs = applicantReceiver } else { - tips.ReceiverAs = 1 + tips.ReceiverAs = adminReceiver } g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.GroupApplicationRejectedNotification, tips) } @@ -459,14 +482,20 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. } opUserID := mcontext.GetOpUserID(ctx) var member map[string]*sdkws.GroupMemberFullInfo - member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID}) + member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID, req.OldOwnerUserID}) if err != nil { return } - tips := &sdkws.GroupOwnerTransferredTips{Group: group, OpUser: member[opUserID], NewGroupOwner: member[req.NewOwnerUserID]} + tips := &sdkws.GroupOwnerTransferredTips{ + Group: group, + OpUser: member[opUserID], + NewGroupOwner: member[req.NewOwnerUserID], + OldGroupOwnerInfo: member[req.OldOwnerUserID], + } if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -480,51 +509,71 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } -func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, groupID, reason string, invitedUserIDList []string) { +func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) } }() - var group *sdkws.GroupInfo - group, err = g.getGroupInfo(ctx, groupID) - if err != nil { - return + + if !g.config.RpcConfig.EnableHistoryForNewMembers { + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) + maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + if err != nil { + return err + } + if _, err = g.msgRpcClient.SetUserConversationsMinSeq(ctx, &msg.SetUserConversationsMinSeqReq{ + UserIDs: entrantUserID, + ConversationID: conversationID, + Seq: maxSeq, + }); err != nil { + return err + } } - var users []*sdkws.GroupMemberFullInfo - users, err = g.getGroupMembers(ctx, groupID, invitedUserIDList) - if err != nil { - return + if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, entrantUserID); err != nil { + return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} - err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) -} -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID string) { - var err error - defer func() { - if err != nil { - log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) - } - }() var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, groupID) if err != nil { - return + return err } - var user *sdkws.GroupMemberFullInfo - user, err = g.getGroupMember(ctx, groupID, entrantUserID) + users, err := g.getGroupMembers(ctx, groupID, entrantUserID) if err != nil { - return + return err } - tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) + + tips := &sdkws.MemberInvitedTips{ + Group: group, + InvitedUserList: users, + } + opUserID := mcontext.GetOpUserID(ctx) + if err = g.fillOpUserByUserID(ctx, opUserID, &tips.OpUser, tips.Group.GroupID); err != nil { + return nil + } + switch { + case invitedOpUserID == "": + case invitedOpUserID == opUserID: + tips.InviterUser = tips.OpUser + default: + if err = g.fillOpUserByUserID(ctx, invitedOpUserID, &tips.InviterUser, tips.Group.GroupID); err != nil { + return err + } + } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) + return nil +} + +func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { + return g.GroupApplicationAgreeMemberEnterNotification(ctx, groupID, "", entrantUserID...) } func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { @@ -564,6 +613,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -588,6 +638,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -615,6 +666,7 @@ func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, gr if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMutedNotification, tips) } @@ -642,6 +694,7 @@ func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupCancelMutedNotification, tips) } @@ -666,6 +719,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setSortVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID, &tips.GroupSortVersion) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -689,6 +743,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -713,5 +768,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go new file mode 100644 index 0000000000..0592aa811c --- /dev/null +++ b/internal/rpc/group/sync.go @@ -0,0 +1,303 @@ +package group + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/constant" + pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" +) + +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { + vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &pbgroup.GetFullGroupMemberUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { + vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(groupIDs) + if req.IdHash == idHash { + groupIDs = nil + } + return &pbgroup.GetFullJoinGroupIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + GroupIDs: groupIDs, + }, nil +} + +func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + group, err := s.db.TakeGroup(ctx, req.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } + var ( + hasGroupUpdate bool + sortVersion uint64 + ) + opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit) + if err != nil { + return nil, err + } + logs := make([]model.VersionLogElem, 0, len(vl.Logs)) + for i, log := range vl.Logs { + switch log.EID { + case model.VersionGroupChangeID: + vl.LogLen-- + hasGroupUpdate = true + case model.VersionSortChangeID: + vl.LogLen-- + sortVersion = uint64(log.Version) + default: + logs = append(logs, vl.Logs[i]) + } + } + vl.Logs = logs + if vl.LogLen > 0 { + hasGroupUpdate = true + } + return vl, nil + }, + CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, + Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + return s.getGroupMembersInfo(ctx, req.GroupID, ids) + }, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + return &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, + } + }, + } + resp, err := opt.Build() + if err != nil { + return nil, err + } + if resp.Full || hasGroupUpdate { + count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + resp.Group = s.groupDB2PB(group, owner.UserID, count) + } + return resp, nil +} + +func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (resp *pbgroup.BatchGetIncrementalGroupMemberResp, err error) { + type VersionInfo struct { + GroupID string + VersionID string + VersionNumber uint64 + } + + var groupIDs []string + + groupsVersionMap := make(map[string]*VersionInfo) + groupsMap := make(map[string]*model.Group) + hasGroupUpdateMap := make(map[string]bool) + sortVersionMap := make(map[string]uint64) + + var targetKeys, versionIDs []string + var versionNumbers []uint64 + + var requestBodyLen int + + for _, group := range req.ReqList { + groupsVersionMap[group.GroupID] = &VersionInfo{ + GroupID: group.GroupID, + VersionID: group.VersionID, + VersionNumber: group.Version, + } + + groupIDs = append(groupIDs, group.GroupID) + } + + groups, err := s.db.FindGroup(ctx, groupIDs) + if err != nil { + return nil, errs.Wrap(err) + } + + for _, group := range groups { + if group.Status == constant.GroupStatusDismissed { + err = servererrs.ErrDismissedAlready.Wrap() + log.ZError(ctx, "This group is Dismissed Already", err, "group is", group.GroupID) + + delete(groupsVersionMap, group.GroupID) + } else { + groupsMap[group.GroupID] = group + } + } + + for groupID, vInfo := range groupsVersionMap { + targetKeys = append(targetKeys, groupID) + versionIDs = append(versionIDs, vInfo.VersionID) + versionNumbers = append(versionNumbers, vInfo.VersionNumber) + } + + opt := incrversion.BatchOption[[]*sdkws.GroupMemberFullInfo, pbgroup.BatchGetIncrementalGroupMemberResp]{ + Ctx: ctx, + TargetKeys: targetKeys, + VersionIDs: versionIDs, + VersionNumbers: versionNumbers, + Versions: func(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { + vLogs, err := s.db.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + for groupID, vlog := range vLogs { + vlogElems := make([]model.VersionLogElem, 0, len(vlog.Logs)) + for i, log := range vlog.Logs { + switch log.EID { + case model.VersionGroupChangeID: + vlog.LogLen-- + hasGroupUpdateMap[groupID] = true + case model.VersionSortChangeID: + vlog.LogLen-- + sortVersionMap[groupID] = uint64(log.Version) + default: + vlogElems = append(vlogElems, vlog.Logs[i]) + } + } + vlog.Logs = vlogElems + if vlog.LogLen > 0 { + hasGroupUpdateMap[groupID] = true + } + } + + return vLogs, nil + }, + CacheMaxVersions: s.db.BatchFindMaxGroupMemberVersionCache, + Find: func(ctx context.Context, groupID string, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + memberInfo, err := s.getGroupMembersInfo(ctx, groupID, ids) + if err != nil { + return nil, err + } + + return memberInfo, err + }, + Resp: func(versions map[string]*model.VersionLog, deleteIdsMap map[string][]string, insertListMap, updateListMap map[string][]*sdkws.GroupMemberFullInfo, fullMap map[string]bool) *pbgroup.BatchGetIncrementalGroupMemberResp { + resList := make(map[string]*pbgroup.GetIncrementalGroupMemberResp) + + for groupID, versionLog := range versions { + resList[groupID] = &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: versionLog.ID.Hex(), + Version: uint64(versionLog.Version), + Full: fullMap[groupID], + Delete: deleteIdsMap[groupID], + Insert: insertListMap[groupID], + Update: updateListMap[groupID], + SortVersion: sortVersionMap[groupID], + } + + requestBodyLen += len(insertListMap[groupID]) + len(updateListMap[groupID]) + len(deleteIdsMap[groupID]) + if requestBodyLen > 200 { + break + } + } + + return &pbgroup.BatchGetIncrementalGroupMemberResp{ + RespList: resList, + } + }, + } + + resp, err = opt.Build() + if err != nil { + return nil, errs.Wrap(err) + } + + for groupID, val := range resp.RespList { + if val.Full || hasGroupUpdateMap[groupID] { + count, err := s.db.FindGroupMemberNum(ctx, groupID) + if err != nil { + return nil, err + } + + owner, err := s.db.TakeGroupOwner(ctx, groupID) + if err != nil { + return nil, err + } + + resp.RespList[groupID].Group = s.groupDB2PB(groupsMap[groupID], owner.UserID, count) + } + } + + return resp, nil + +} + +func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: s.db.FindJoinIncrVersion, + CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, + Find: s.getGroupsInfo, + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + return &pbgroup.GetIncrementalJoinGroupResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/incrversion/batch_option.go b/internal/rpc/incrversion/batch_option.go new file mode 100644 index 0000000000..34d1b25066 --- /dev/null +++ b/internal/rpc/incrversion/batch_option.go @@ -0,0 +1,207 @@ +package incrversion + +import ( + "context" + "fmt" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type BatchOption[A, B any] struct { + Ctx context.Context + TargetKeys []string + VersionIDs []string + VersionNumbers []uint64 + //SyncLimit int + Versions func(ctx context.Context, dIds []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) + CacheMaxVersions func(ctx context.Context, dIds []string) (map[string]*model.VersionLog, error) + Find func(ctx context.Context, dId string, ids []string) (A, error) + Resp func(versionsMap map[string]*model.VersionLog, deleteIdsMap map[string][]string, insertListMap, updateListMap map[string]A, fullMap map[string]bool) *B +} + +func (o *BatchOption[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *BatchOption[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") + } + if len(o.TargetKeys) == 0 { + return o.newError("targetKeys is empty") + } + if o.Versions == nil { + return o.newError("func versions is nil") + } + if o.Find == nil { + return o.newError("func find is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil +} + +func (o *BatchOption[A, B]) validVersions() []bool { + valids := make([]bool, len(o.VersionIDs)) + for i, versionID := range o.VersionIDs { + objID, err := primitive.ObjectIDFromHex(versionID) + valids[i] = (err == nil && (!objID.IsZero()) && o.VersionNumbers[i] > 0) + } + return valids +} + +func (o *BatchOption[A, B]) equalIDs(objIDs []primitive.ObjectID) []bool { + equals := make([]bool, len(o.VersionIDs)) + for i, versionID := range o.VersionIDs { + equals[i] = versionID == objIDs[i].Hex() + } + return equals +} + +func (o *BatchOption[A, B]) getVersions(tags *[]int) (versions map[string]*model.VersionLog, err error) { + var dIDs []string + var versionNums []uint64 + var limits []int + + valids := o.validVersions() + + if o.CacheMaxVersions == nil { + for i, valid := range valids { + if valid { + (*tags)[i] = tagQuery + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, o.VersionNumbers[i]) + limits = append(limits, syncLimit) + } else { + (*tags)[i] = tagFull + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, 0) + limits = append(limits, 0) + } + } + + versions, err = o.Versions(o.Ctx, dIDs, versionNums, limits) + if err != nil { + return nil, errs.Wrap(err) + } + return versions, nil + + } else { + caches, err := o.CacheMaxVersions(o.Ctx, o.TargetKeys) + if err != nil { + return nil, errs.Wrap(err) + } + + objIDs := make([]primitive.ObjectID, len(o.VersionIDs)) + + for i, versionID := range o.VersionIDs { + objID, _ := primitive.ObjectIDFromHex(versionID) + objIDs[i] = objID + } + + equals := o.equalIDs(objIDs) + for i, valid := range valids { + if !valid { + (*tags)[i] = tagFull + } else if !equals[i] { + (*tags)[i] = tagFull + } else if o.VersionNumbers[i] == uint64(caches[o.TargetKeys[i]].Version) { + (*tags)[i] = tagEqual + } else { + (*tags)[i] = tagQuery + dIDs = append(dIDs, o.TargetKeys[i]) + versionNums = append(versionNums, o.VersionNumbers[i]) + limits = append(limits, syncLimit) + + delete(caches, o.TargetKeys[i]) + } + } + + if dIDs != nil { + versionMap, err := o.Versions(o.Ctx, dIDs, versionNums, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + for k, v := range versionMap { + caches[k] = v + } + } + + versions = caches + } + return versions, nil +} + +func (o *BatchOption[A, B]) Build() (*B, error) { + if err := o.check(); err != nil { + return nil, errs.Wrap(err) + } + + tags := make([]int, len(o.TargetKeys)) + versions, err := o.getVersions(&tags) + if err != nil { + return nil, errs.Wrap(err) + } + + fullMap := make(map[string]bool) + for i, tag := range tags { + switch tag { + case tagQuery: + vLog := versions[o.TargetKeys[i]] + fullMap[o.TargetKeys[i]] = vLog.ID.Hex() != o.VersionIDs[i] || uint64(vLog.Version) < o.VersionNumbers[i] || len(vLog.Logs) != vLog.LogLen + case tagFull: + fullMap[o.TargetKeys[i]] = true + case tagEqual: + fullMap[o.TargetKeys[i]] = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } + } + + var ( + insertIdsMap = make(map[string][]string) + deleteIdsMap = make(map[string][]string) + updateIdsMap = make(map[string][]string) + ) + + for _, targetKey := range o.TargetKeys { + if !fullMap[targetKey] { + version := versions[targetKey] + insertIds, deleteIds, updateIds := version.DeleteAndChangeIDs() + insertIdsMap[targetKey] = insertIds + deleteIdsMap[targetKey] = deleteIds + updateIdsMap[targetKey] = updateIds + } + } + + var ( + insertListMap = make(map[string]A) + updateListMap = make(map[string]A) + ) + + for targetKey, insertIds := range insertIdsMap { + if len(insertIds) > 0 { + insertList, err := o.Find(o.Ctx, targetKey, insertIds) + if err != nil { + return nil, errs.Wrap(err) + } + insertListMap[targetKey] = insertList + } + } + + for targetKey, updateIds := range updateIdsMap { + if len(updateIds) > 0 { + updateList, err := o.Find(o.Ctx, targetKey, updateIds) + if err != nil { + return nil, errs.Wrap(err) + } + updateListMap[targetKey] = updateList + } + } + + return o.Resp(versions, deleteIdsMap, insertListMap, updateListMap, fullMap), nil +} diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go new file mode 100644 index 0000000000..af1200d5c0 --- /dev/null +++ b/internal/rpc/incrversion/option.go @@ -0,0 +1,153 @@ +package incrversion + +import ( + "context" + "fmt" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +//func Limit(maxSync int, version uint64) int { +// if version == 0 { +// return 0 +// } +// return maxSync +//} + +const syncLimit = 200 + +const ( + tagQuery = iota + 1 + tagFull + tagEqual +) + +type Option[A, B any] struct { + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + //SyncLimit int + CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) + Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + //SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + Resp func(version *model.VersionLog, deleteIds []string, insertList, updateList []A, full bool) *B +} + +func (o *Option[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *Option[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") + } + if o.VersionKey == "" { + return o.newError("versionKey is empty") + } + //if o.SyncLimit <= 0 { + // return o.newError("invalid synchronization quantity") + //} + if o.Version == nil { + return o.newError("func version is nil") + } + //if o.SortID == nil { + // return o.newError("func allID is nil") + //} + if o.Find == nil { + return o.newError("func find is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil +} + +func (o *Option[A, B]) validVersion() bool { + objID, err := primitive.ObjectIDFromHex(o.VersionID) + return err == nil && (!objID.IsZero()) && o.VersionNumber > 0 +} + +func (o *Option[A, B]) equalID(objID primitive.ObjectID) bool { + return o.VersionID == objID.Hex() +} + +func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { + if o.CacheMaxVersion == nil { + if o.validVersion() { + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) + } + *tag = tagFull + return o.Version(o.Ctx, o.VersionKey, 0, 0) + } else { + cache, err := o.CacheMaxVersion(o.Ctx, o.VersionKey) + if err != nil { + return nil, err + } + if !o.validVersion() { + *tag = tagFull + return cache, nil + } + if !o.equalID(cache.ID) { + *tag = tagFull + return cache, nil + } + if o.VersionNumber == uint64(cache.Version) { + *tag = tagEqual + return cache, nil + } + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) + } +} + +func (o *Option[A, B]) Build() (*B, error) { + if err := o.check(); err != nil { + return nil, err + } + var tag int + version, err := o.getVersion(&tag) + if err != nil { + return nil, err + } + var full bool + switch tag { + case tagQuery: + full = version.ID.Hex() != o.VersionID || uint64(version.Version) < o.VersionNumber || len(version.Logs) != version.LogLen + case tagFull: + full = true + case tagEqual: + full = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } + var ( + insertIds []string + deleteIds []string + updateIds []string + ) + if !full { + insertIds, deleteIds, updateIds = version.DeleteAndChangeIDs() + } + var ( + insertList []A + updateList []A + ) + if len(insertIds) > 0 { + insertList, err = o.Find(o.Ctx, insertIds) + if err != nil { + return nil, err + } + } + if len(updateIds) > 0 { + updateList, err = o.Find(o.Ctx, updateIds) + if err != nil { + return nil, err + } + } + return o.Resp(version, deleteIds, insertList, updateList, full), nil +} diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index 10404675eb..3b76c25537 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -41,6 +41,7 @@ func toCommonCallback(ctx context.Context, msg *pbchat.SendMsgReq, command strin MsgFrom: msg.MsgData.MsgFrom, ContentType: msg.MsgData.ContentType, Status: msg.MsgData.Status, + SendTime: msg.MsgData.SendTime, CreateTime: msg.MsgData.CreateTime, AtUserIDList: msg.MsgData.AtUserIDList, SenderFaceURL: msg.MsgData.SenderFaceURL, @@ -83,6 +84,11 @@ func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config if msg.MsgData.ContentType == constant.Typing { return } + // According to the attentionIds configuration, only some users are sent + attentionIds := after.AttentionIds + if attentionIds != nil && !datautil.Contain(msg.MsgData.RecvID, attentionIds...) && !datautil.Contain(msg.MsgData.SendID, attentionIds...) { + return + } cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), RecvID: msg.MsgData.RecvID, diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 774eae32cd..c5bd36b445 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,16 +2,22 @@ package msg import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "strings" - "time" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/idutil" + "github.com/openimsdk/tools/utils/stringutil" + "golang.org/x/sync/errgroup" ) +// hard delete in Database. func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -24,26 +30,21 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. msgNum int start = time.Now() ) + clearMsg := func(ctx context.Context) (bool, error) { - conversationSeqs := make(map[string]struct{}) - defer func() { - req := &conversation.UpdateConversationReq{ - MsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), - } - for conversationID := range conversationSeqs { - req.ConversationID = conversationID - if err := m.Conversation.UpdateConversations(ctx, req); err != nil { - log.ZError(ctx, "update conversation max seq failed", err, "conversationID", conversationID, "msgDestructTime", req.MsgDestructTime) - } - } - }() - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, 100) + docIDs, err := m.MsgDatabase.GetDocIDs(ctx) + if err != nil { + return false, err + } + + msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, 5000) if err != nil { return false, err } if len(msgs) == 0 { return false, nil } + for _, msg := range msgs { index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) if err != nil { @@ -52,26 +53,73 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. if len(index) == 0 { return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") } + docNum++ msgNum += len(index) - conversationID := msg.DocID[:strings.LastIndex(msg.DocID, ":")] - if _, ok := conversationSeqs[conversationID]; !ok { - conversationSeqs[conversationID] = struct{}{} - } } + return true, nil } - for { - keep, err := clearMsg(ctx) - if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return nil, err - } - if !keep { - log.ZInfo(ctx, "clear msg success", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - break - } - log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + + _, err = clearMsg(ctx) + if err != nil { + log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + return nil, err } + + log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + return &msg.ClearMsgResp{}, nil } + +// soft delete for self +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { + temp := convert.ConversationsPb2DB(req.Conversations) + + batchNum := 100 + + errg, _ := errgroup.WithContext(ctx) + errg.SetLimit(100) + + for i := 0; i < len(temp); i += batchNum { + batch := temp[i:min(i+batchNum, len(temp))] + + errg.Go(func() error { + for _, conversation := range batch { + handleCtx := mcontext.NewCtx(stringutil.GetSelfFuncName() + "-" + idutil.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID) + log.ZDebug(handleCtx, "User MsgsDestruct", + "conversationID", conversation.ConversationID, + "ownerUserID", conversation.OwnerUserID, + "msgDestructTime", conversation.MsgDestructTime, + "lastMsgDestructTime", conversation.LatestMsgDestructTime) + + seqs, err := m.MsgDatabase.UserMsgsDestruct(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) + if err != nil { + log.ZError(handleCtx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) + continue + } + + if len(seqs) > 0 { + if err := m.Conversation.UpdateConversation(handleCtx, + &pbconversation.UpdateConversationReq{ + UserIDs: []string{conversation.OwnerUserID}, + ConversationID: conversation.ConversationID, + LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli())}); err != nil { + log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) + continue + } + + // if you need Notify SDK client userseq is update. + // m.msgNotificationSender.UserDeleteMsgsNotification(handleCtx, conversation.OwnerUserID, conversation.ConversationID, seqs) + } + } + return nil + }) + } + + if err := errg.Wait(); err != nil { + return nil, err + } + + return nil, nil +} diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 27465c2105..4d9eb6db9c 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -16,13 +16,15 @@ package msg import ( "context" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" pbmsg "github.com/openimsdk/protocol/msg" ) func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil { + if err != nil && errs.Unwrap(err) != redis.Nil { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil @@ -51,3 +53,12 @@ func (m *msgServer) GetMsgByConversationIDs(ctx context.Context, req *pbmsg.GetM } return &pbmsg.GetMsgByConversationIDsResp{MsgDatas: Msgs}, nil } + +func (m *msgServer) SetUserConversationsMinSeq(ctx context.Context, req *pbmsg.SetUserConversationsMinSeqReq) (*pbmsg.SetUserConversationsMinSeqResp, error) { + for _, userID := range req.UserIDs { + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, map[string]int64{req.ConversationID: req.Seq}); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationsMinSeqResp{}, nil +} diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index f1fb28ffff..91f41f1b11 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -50,6 +51,7 @@ type ( ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data. Handlers MessageInterceptorChain // Chain of handlers for processing messages. notificationSender *rpcclient.NotificationSender // RPC client for sending notifications. + msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. config *Config // Global configuration settings. webhookClient *webhook.Client } @@ -86,12 +88,21 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } @@ -108,7 +119,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } s.notificationSender = rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithLocalSendMsg(s.SendMsg)) + s.msgNotificationSender = NewMsgNotificationSender(config, rpcclient.WithLocalSendMsg(s.SendMsg)) + msg.RegisterMsgServer(server, s) + return nil } diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index afb79506ee..ea4c487b9e 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -16,16 +16,15 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/timeutil" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/timeutil" ) func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessageBySeqsReq) (*sdkws.PullMessageBySeqsResp, error) { @@ -86,6 +85,35 @@ func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessag return resp, nil } +func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + resp := &msg.GetSeqMessageResp{ + Msgs: make(map[string]*sdkws.PullMsgs), + NotificationMsgs: make(map[string]*sdkws.PullMsgs), + } + for _, conv := range req.Conversations { + _, _, msgs, err := m.MsgDatabase.GetMsgBySeqs(ctx, req.UserID, conv.ConversationID, conv.Seqs) + if err != nil { + return nil, err + } + var pullMsgs *sdkws.PullMsgs + if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { + pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.NotificationMsgs[conv.ConversationID] = pullMsgs + } + } else { + pullMsgs, ok = resp.Msgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.Msgs[conv.ConversationID] = pullMsgs + } + } + pullMsgs.Msgs = append(pullMsgs.Msgs, msgs...) + } + return resp, nil +} + func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -104,6 +132,12 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd log.ZWarn(ctx, "GetMaxSeqs error", err, "conversationIDs", conversationIDs, "maxSeqs", maxSeqs) return nil, err } + // avoid pulling messages from sessions with a large number of max seq values of 0 + for conversationID, seq := range maxSeqs { + if seq == 0 { + delete(maxSeqs, conversationID) + } + } resp := new(sdkws.GetMaxSeqResp) resp.MaxSeqs = maxSeqs return resp, nil @@ -111,7 +145,7 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (resp *msg.SearchMessageResp, err error) { var chatLogs []*sdkws.MsgData - var total int32 + var total int64 resp = &msg.SearchMessageResp{} if total, chatLogs, err = m.MsgDatabase.SearchMessage(ctx, req); err != nil { return nil, err @@ -194,7 +228,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq } resp.ChatLogs = append(resp.ChatLogs, pbchatLog) } - resp.ChatLogsNum = total + resp.ChatLogsNum = int32(total) return resp, nil } diff --git a/internal/rpc/friend/black.go b/internal/rpc/relation/black.go similarity index 79% rename from internal/rpc/friend/black.go rename to internal/rpc/relation/black.go index caec08b7ac..e149e31653 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/relation/black.go @@ -12,20 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) -func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.GetPaginationBlacksReq) (resp *pbfriend.GetPaginationBlacksResp, err error) { +func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.GetPaginationBlacksReq) (resp *relation.GetPaginationBlacksResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -33,7 +34,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge if err != nil { return nil, err } - resp = &pbfriend.GetPaginationBlacksResp{} + resp = &relation.GetPaginationBlacksResp{} resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -42,18 +43,18 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge return resp, nil } -func (s *friendServer) IsBlack(ctx context.Context, req *pbfriend.IsBlackReq) (*pbfriend.IsBlackResp, error) { +func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (*relation.IsBlackResp, error) { in1, in2, err := s.blackDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } - resp := &pbfriend.IsBlackResp{} + resp := &relation.IsBlackResp{} resp.InUser1Blacks = in1 resp.InUser2Blacks = in2 return resp, nil } -func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlackReq) (*pbfriend.RemoveBlackResp, error) { +func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlackReq) (*relation.RemoveBlackResp, error) { if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -64,10 +65,10 @@ func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlac s.notificationSender.BlackDeletedNotification(ctx, req) - return &pbfriend.RemoveBlackResp{}, nil + return &relation.RemoveBlackResp{}, nil } -func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) (*pbfriend.AddBlackResp, error) { +func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) (*relation.AddBlackResp, error) { if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -87,5 +88,5 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) return nil, err } s.notificationSender.BlackAddedNotification(ctx, req) - return &pbfriend.AddBlackResp{}, nil + return &relation.AddBlackResp{}, nil } diff --git a/internal/rpc/friend/callback.go b/internal/rpc/relation/callback.go similarity index 88% rename from internal/rpc/friend/callback.go rename to internal/rpc/relation/callback.go index 0610cdb78a..69c4c9e0ec 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/relation/callback.go @@ -12,18 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" ) -func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { +func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *relation.DeleteFriendReq) { cbReq := &cbapi.CallbackAfterDeleteFriendReq{ CallbackCommand: cbapi.CallbackAfterDeleteFriendCommand, OwnerUserID: req.OwnerUserID, @@ -32,7 +33,7 @@ func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *conf s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterDeleteFriendResp{}, after) } -func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ApplyToAddFriendReq) error { +func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *relation.ApplyToAddFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendCommand, @@ -50,7 +51,7 @@ func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *confi }) } -func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.ApplyToAddFriendReq) { +func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *relation.ApplyToAddFriendReq) { cbReq := &cbapi.CallbackAfterAddFriendReq{ CallbackCommand: cbapi.CallbackAfterAddFriendCommand, FromUserID: req.FromUserID, @@ -61,8 +62,7 @@ func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config. s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *pbfriend.SetFriendRemarkReq) { - +func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *relation.SetFriendRemarkReq) { cbReq := &cbapi.CallbackAfterSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackAfterSetFriendRemarkCommand, OwnerUserID: req.OwnerUserID, @@ -73,7 +73,7 @@ func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *c s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *pbfriend.ImportFriendReq) { +func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *relation.ImportFriendReq) { cbReq := &cbapi.CallbackAfterImportFriendsReq{ CallbackCommand: cbapi.CallbackAfterImportFriendsCommand, OwnerUserID: req.OwnerUserID, @@ -83,7 +83,7 @@ func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *con s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *pbfriend.RemoveBlackReq) { +func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *relation.RemoveBlackReq) { cbReq := &cbapi.CallbackAfterRemoveBlackReq{ CallbackCommand: cbapi.CallbackAfterRemoveBlackCommand, OwnerUserID: req.OwnerUserID, @@ -93,7 +93,7 @@ func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *confi s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *pbfriend.SetFriendRemarkReq) error { +func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *relation.SetFriendRemarkReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackBeforeSetFriendRemarkCommand, @@ -112,7 +112,7 @@ func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before }) } -func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *pbfriend.AddBlackReq) error { +func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *relation.AddBlackReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddBlackReq{ CallbackCommand: cbapi.CallbackBeforeAddBlackCommand, @@ -124,7 +124,7 @@ func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config }) } -func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *pbfriend.RespondFriendApplyReq) error { +func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *relation.RespondFriendApplyReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendAgreeReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendAgreeCommand, @@ -138,7 +138,7 @@ func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before * }) } -func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ImportFriendReq) error { +func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *relation.ImportFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeImportFriendsReq{ CallbackCommand: cbapi.CallbackBeforeImportFriendsCommand, diff --git a/internal/rpc/friend/friend.go b/internal/rpc/relation/friend.go similarity index 71% rename from internal/rpc/friend/friend.go rename to internal/rpc/relation/friend.go index 8b2dea995f..9130589321 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/relation/friend.go @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" + + "github.com/openimsdk/tools/mq/memamq" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -30,7 +33,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -40,7 +43,7 @@ import ( ) type friendServer struct { - friendDatabase controller.FriendDatabase + db controller.FriendDatabase blackDatabase controller.BlackDatabase userRpcClient *rpcclient.UserRpcClient notificationSender *FriendNotificationSender @@ -48,13 +51,14 @@ type friendServer struct { RegisterCenter discovery.SvcDiscoveryRegistry config *Config webhookClient *webhook.Client + queue *memamq.MemoryQueue } type Config struct { RpcConfig config.Friend RedisConfig config.Redis MongodbConfig config.Mongo - //ZookeeperConfig config.ZooKeeper + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks @@ -100,8 +104,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) // Register Friend server with refactored MongoDB and Redis integrations - pbfriend.RegisterFriendServer(server, &friendServer{ - friendDatabase: controller.NewFriendDatabase( + relation.RegisterFriendServer(server, &friendServer{ + db: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB, redis.GetRocksCacheOptions()), @@ -117,14 +121,14 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + queue: memamq.NewMemoryQueue(16, 1024*1024), }) - return nil } // ok. -func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) (resp *pbfriend.ApplyToAddFriendResp, err error) { - resp = &pbfriend.ApplyToAddFriendResp{} +func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.ApplyToAddFriendReq) (resp *relation.ApplyToAddFriendResp, err error) { + resp = &relation.ApplyToAddFriendResp{} if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -138,14 +142,14 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply return nil, err } - in1, in2, err := s.friendDatabase.CheckIn(ctx, req.FromUserID, req.ToUserID) + in1, in2, err := s.db.CheckIn(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } if in1 && in2 { return nil, servererrs.ErrRelationshipAlready.WrapMsg("already friends has f") } - if err = s.friendDatabase.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { + if err = s.db.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { return nil, err } s.notificationSender.FriendApplicationAddNotification(ctx, req) @@ -154,7 +158,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply } // ok. -func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFriendReq) (resp *pbfriend.ImportFriendResp, err error) { +func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFriendReq) (resp *relation.ImportFriendResp, err error) { if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -172,11 +176,11 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr return nil, err } - if err := s.friendDatabase.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { + if err := s.db.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { return nil, err } for _, userID := range req.FriendUserIDs { - s.notificationSender.FriendApplicationAgreedNotification(ctx, &pbfriend.RespondFriendApplyReq{ + s.notificationSender.FriendApplicationAgreedNotification(ctx, &relation.RespondFriendApplyReq{ FromUserID: req.OwnerUserID, ToUserID: userID, HandleResult: constant.FriendResponseAgree, @@ -184,12 +188,12 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr } s.webhookAfterImportFriends(ctx, &s.config.WebhooksConfig.AfterImportFriends, req) - return &pbfriend.ImportFriendResp{}, nil + return &relation.ImportFriendResp{}, nil } // ok. -func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.RespondFriendApplyReq) (resp *pbfriend.RespondFriendApplyResp, err error) { - resp = &pbfriend.RespondFriendApplyResp{} +func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.RespondFriendApplyReq) (resp *relation.RespondFriendApplyResp, err error) { + resp = &relation.RespondFriendApplyResp{} if err := authverify.CheckAccessV3(ctx, req.ToUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -204,7 +208,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res if err := s.webhookBeforeAddFriendAgree(ctx, &s.config.WebhooksConfig.BeforeAddFriendAgree, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - err := s.friendDatabase.AgreeFriendRequest(ctx, &friendRequest) + err := s.db.AgreeFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -212,7 +216,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { - err := s.friendDatabase.RefuseFriendRequest(ctx, &friendRequest) + err := s.db.RefuseFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -223,16 +227,16 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res } // ok. -func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFriendReq) (resp *pbfriend.DeleteFriendResp, err error) { - resp = &pbfriend.DeleteFriendResp{} +func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFriendReq) (resp *relation.DeleteFriendResp, err error) { + resp = &relation.DeleteFriendResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { + if err := s.db.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { return nil, err } s.notificationSender.FriendDeletedNotification(ctx, req) @@ -241,19 +245,19 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFri } // ok. -func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFriendRemarkReq) (resp *pbfriend.SetFriendRemarkResp, err error) { +func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFriendRemarkReq) (resp *relation.SetFriendRemarkResp, err error) { if err = s.webhookBeforeSetFriendRemark(ctx, &s.config.WebhooksConfig.BeforeSetFriendRemark, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - resp = &pbfriend.SetFriendRemarkResp{} + resp = &relation.SetFriendRemarkResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { + if err := s.db.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { return nil, err } s.webhookAfterSetFriendRemark(ctx, &s.config.WebhooksConfig.AfterSetFriendRemark, req) @@ -262,29 +266,40 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFri } // ok. -func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.GetDesignatedFriendsReq) (resp *pbfriend.GetDesignatedFriendsResp, err error) { - resp = &pbfriend.GetDesignatedFriendsResp{} +func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { + resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + friends, err := s.getFriend(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } - if resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap); err != nil { + return &relation.GetDesignatedFriendsResp{ + FriendsInfo: friends, + }, nil +} + +func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friendUserIDs []string) ([]*sdkws.FriendInfo, error) { + if len(friendUserIDs) == 0 { + return nil, nil + } + friends, err := s.db.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + if err != nil { return nil, err } - return resp, nil + return convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, - req *pbfriend.GetDesignatedFriendsApplyReq) (resp *pbfriend.GetDesignatedFriendsApplyResp, err error) { - friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) + req *relation.GetDesignatedFriendsApplyReq, +) (resp *relation.GetDesignatedFriendsApplyResp, err error) { + friendRequests, err := s.db.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } - resp = &pbfriend.GetDesignatedFriendsApplyResp{} + resp = &relation.GetDesignatedFriendsApplyResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -293,29 +308,33 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, } // Get received friend requests (i.e., those initiated by others). -func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyToReq) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) { +func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *relation.GetPaginationFriendsApplyToReq) (resp *relation.GetPaginationFriendsApplyToResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) + + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsApplyToResp{} + + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err } + resp.Total = int32(total) + return resp, nil } -func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyFromReq) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) { - resp = &pbfriend.GetPaginationFriendsApplyFromResp{} +func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { + resp = &relation.GetPaginationFriendsApplyFromResp{} if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -328,24 +347,24 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *p } // ok. -func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) (resp *pbfriend.IsFriendResp, err error) { - resp = &pbfriend.IsFriendResp{} - resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) +func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { + resp = &relation.IsFriendResp{} + resp.InUser1Friends, resp.InUser2Friends, err = s.db.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } return resp, nil } -func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.GetPaginationFriendsReq) (resp *pbfriend.GetPaginationFriendsResp, err error) { +func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.GetPaginationFriendsReq) (resp *relation.GetPaginationFriendsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + total, friends, err := s.db.PageOwnerFriends(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsResp{} + resp = &relation.GetPaginationFriendsResp{} resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -354,19 +373,19 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G return resp, nil } -func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriendIDsReq) (resp *pbfriend.GetFriendIDsResp, err error) { +func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriendIDsReq) (resp *relation.GetFriendIDsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - resp = &pbfriend.GetFriendIDsResp{} - resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) + resp = &relation.GetFriendIDsResp{} + resp.FriendIDs, err = s.db.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } return resp, nil } -func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfriend.GetSpecifiedFriendsInfoReq) (*pbfriend.GetSpecifiedFriendsInfoResp, error) { +func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relation.GetSpecifiedFriendsInfoReq) (*relation.GetSpecifiedFriendsInfoResp, error) { if len(req.UserIDList) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDList is empty") } @@ -377,7 +396,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien if err != nil { return nil, err } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) if err != nil { return nil, err } @@ -391,8 +410,8 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien blackMap := datautil.SliceToMap(blacks, func(e *model.Black) string { return e.BlockUserID }) - resp := &pbfriend.GetSpecifiedFriendsInfoResp{ - Infos: make([]*pbfriend.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), + resp := &relation.GetSpecifiedFriendsInfoResp{ + Infos: make([]*relation.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), } for _, userID := range req.UserIDList { user := userMap[userID] @@ -401,7 +420,6 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } var friendInfo *sdkws.FriendInfo if friend := friendMap[userID]; friend != nil { - friendInfo = &sdkws.FriendInfo{ OwnerUserID: friend.OwnerUserID, Remark: friend.Remark, @@ -422,7 +440,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien Ex: black.Ex, } } - resp.Infos = append(resp.Infos, &pbfriend.GetSpecifiedFriendsInfoInfo{ + resp.Infos = append(resp.Infos, &relation.GetSpecifiedFriendsInfoInfo{ UserInfo: user, FriendInfo: friendInfo, BlackInfo: blackInfo, @@ -430,10 +448,11 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, - req *pbfriend.UpdateFriendsReq, -) (*pbfriend.UpdateFriendsResp, error) { + req *relation.UpdateFriendsReq, +) (*relation.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("friendIDList is empty") } @@ -441,7 +460,7 @@ func (s *friendServer) UpdateFriends( return nil, errs.ErrArgs.WrapMsg("friendIDList repeated") } - _, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + _, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } @@ -457,12 +476,27 @@ func (s *friendServer) UpdateFriends( if req.Ex != nil { val["ex"] = req.Ex.Value } - if err = s.friendDatabase.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { + if err = s.db.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { return nil, err } - resp := &pbfriend.UpdateFriendsResp{} + resp := &relation.UpdateFriendsResp{} s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } + +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *relation.GetIncrementalFriendsApplyToReq) (*relation.GetIncrementalFriendsApplyToResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *relation.GetIncrementalFriendsApplyFromReq) (*relation.GetIncrementalFriendsApplyFromResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *relation.GetIncrementalBlacksReq) (*relation.GetIncrementalBlacksResp, error) { + // TODO implement me + return nil, nil +} diff --git a/internal/rpc/friend/notification.go b/internal/rpc/relation/notification.go similarity index 83% rename from internal/rpc/friend/notification.go rename to internal/rpc/relation/notification.go index 8089a9bdc8..83c5d2ca9d 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/relation/notification.go @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -24,7 +27,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) @@ -127,7 +130,7 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) { +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, ToUserID: req.ToUserID, @@ -137,7 +140,7 @@ func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context. func (f *FriendNotificationSender) FriendApplicationAgreedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -148,7 +151,7 @@ func (f *FriendNotificationSender) FriendApplicationAgreedNotification( func (f *FriendNotificationSender) FriendApplicationRefusedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -182,7 +185,7 @@ func (f *FriendNotificationSender) FriendAddedNotification( return nil } -func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *pbfriend.DeleteFriendReq) { +func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *relation.DeleteFriendReq) { tips := sdkws.FriendDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.FriendUserID, @@ -190,10 +193,37 @@ func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context f.Notification(ctx, req.OwnerUserID, req.FriendUserID, constant.FriendDeletedNotification, &tips) } +func (f *FriendNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + return + } + } +} + +func (f *FriendNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() + for _, elem := range coll.Doc.Logs { + if elem.EID == relationtb.VersionSortChangeID { + *sortVersion = uint64(elem.Version) + } + } + } + } +} + func (f *FriendNotificationSender) FriendRemarkSetNotification(ctx context.Context, fromUserID, toUserID string) { tips := sdkws.FriendInfoChangedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = fromUserID tips.FromToUserID.ToUserID = toUserID + f.setSortVersion(ctx, &tips.FriendVersion, &tips.FriendVersionID, database.FriendVersionName, toUserID, &tips.FriendSortVersion) f.Notification(ctx, fromUserID, toUserID, constant.FriendRemarkSetNotification, &tips) } @@ -204,14 +234,14 @@ func (f *FriendNotificationSender) FriendsInfoUpdateNotification(ctx context.Con f.Notification(ctx, toUserID, toUserID, constant.FriendsInfoUpdateNotification, &tips) } -func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *pbfriend.AddBlackReq) { +func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *relation.AddBlackReq) { tips := sdkws.BlackAddedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = req.OwnerUserID tips.FromToUserID.ToUserID = req.BlackUserID f.Notification(ctx, req.OwnerUserID, req.BlackUserID, constant.BlackAddedNotification, &tips) } -func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *pbfriend.RemoveBlackReq) { +func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *relation.RemoveBlackReq) { blackDeletedTips := sdkws.BlackDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.BlackUserID, diff --git a/internal/rpc/relation/sync.go b/internal/rpc/relation/sync.go new file mode 100644 index 0000000000..0ad94fe825 --- /dev/null +++ b/internal/rpc/relation/sync.go @@ -0,0 +1,104 @@ +package relation + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "slices" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/relation" +) + +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + if len(userIDs) > 0 { + friendUserIDs := []string{req.UserID} + noCancelCtx := context.WithoutCancel(ctx) + err := s.queue.PushCtx(ctx, func() { + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(noCancelCtx, userID, friendUserIDs, model.VersionStateUpdate); err != nil { + log.ZError(ctx, "OwnerIncrVersion", err, "userID", userID, "friendUserIDs", friendUserIDs) + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(noCancelCtx, req.UserID, userID) + } + }) + if err != nil { + log.ZError(ctx, "NotificationUserInfoUpdate timeout", err, "userID", req.UserID) + } + } + return &relation.NotificationUserInfoUpdateResp{}, nil +} + +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &relation.GetFullFriendUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + var sortVersion uint64 + opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindFriendIncrVersion(ctx, ownerUserID, version, limit) + if err != nil { + return nil, err + } + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == model.VersionSortChangeID { + vl.LogLen-- + sortVersion = uint64(elem.Version) + return true + } + return false + }) + return vl, nil + }, + CacheMaxVersion: s.db.FindMaxFriendVersionCache, + Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { + return s.getFriend(ctx, req.UserID, ids) + }, + Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, + SortVersion: sortVersion, + } + }, + } + return opt.Build() +} diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 5c0b1f2e6b..68d7088b0c 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -26,7 +26,6 @@ import ( "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/stringutil" ) func genLogID() string { @@ -50,13 +49,14 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) platform := constant.PlatformID2Name[int(req.Platform)] for _, fileURL := range req.FileURLs { log := relationtb.Log{ - Version: req.Version, - SystemType: req.SystemType, Platform: platform, UserID: userID, CreateTime: time.Now(), Url: fileURL.URL, FileName: fileURL.Filename, + SystemType: req.SystemType, + Version: req.Version, + Ex: req.Ex, } for i := 0; i < 20; i++ { id := genLogID() @@ -110,7 +110,7 @@ func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo { return &third.LogInfo{ Filename: log.FileName, UserID: log.UserID, - Platform: stringutil.StringToInt32(log.Platform), + Platform: log.Platform, Url: log.Url, CreateTime: log.CreateTime.UnixMilli(), LogID: log.LogID, diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 4cb1b81d06..fb6a1157e1 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,13 +19,17 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -283,6 +287,53 @@ func (t *thirdServer) apiAddress(prefix, name string) string { return prefix + name } +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + var conf config.Third + expireTime := time.UnixMilli(req.ExpireTime) + var deltotal int + findPagination := &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 1000, + } + for { + total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 && t.minio != nil { + thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) + + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) + t.s3dataBase.DeleteObject(ctx, key) + } + } + for _, model := range models { + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + if err != nil { + return nil, errs.Wrap(err) + } + } + if total < int64(findPagination.ShowNumber) { + break + } + deltotal += int(total) + } + log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", deltotal) + return &third.DeleteOutdatedDataResp{}, nil +} + type FormDataMate struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 7560486a00..0eeaaa3140 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -31,6 +31,7 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" "google.golang.org/grpc" @@ -42,7 +43,9 @@ type thirdServer struct { userRpcClient rpcclient.UserRpcClient defaultExpire time.Duration config *Config + minio *minio.Minio } + type Config struct { RpcConfig config.Third RedisConfig config.Redis @@ -73,14 +76,20 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable - var o s3.Interface + var ( + o s3.Interface + minioCli *minio.Minio + ) switch enable { case "minio": - o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + o = minioCli case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": o, err = oss.NewOSS(*config.RpcConfig.Object.Oss.Build()) + case "kodo": + o, err = kodo.NewKodo(*config.RpcConfig.Object.Kodo.Build()) default: err = fmt.Errorf("invalid object enable: %s", enable) } @@ -94,10 +103,15 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, + minio: minioCli, }) return nil } +func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { + return t.minio.GetImageThumbnailKey(ctx, name) +} + func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go new file mode 100644 index 0000000000..4e7823306f --- /dev/null +++ b/internal/rpc/user/online.go @@ -0,0 +1,103 @@ +package user + +import ( + "context" + "github.com/openimsdk/tools/utils/datautil" + + "github.com/openimsdk/protocol/constant" + pbuser "github.com/openimsdk/protocol/user" +) + +func (s *userServer) getUserOnlineStatus(ctx context.Context, userID string) (*pbuser.OnlineStatus, error) { + platformIDs, err := s.online.GetOnline(ctx, userID) + if err != nil { + return nil, err + } + status := pbuser.OnlineStatus{ + UserID: userID, + PlatformIDs: platformIDs, + } + if len(platformIDs) > 0 { + status.Status = constant.Online + } else { + status.Status = constant.Offline + } + return &status, nil +} + +func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) ([]*pbuser.OnlineStatus, error) { + res := make([]*pbuser.OnlineStatus, 0, len(userIDs)) + for _, userID := range userIDs { + status, err := s.getUserOnlineStatus(ctx, userID) + if err != nil { + return nil, err + } + res = append(res, status) + } + return res, nil +} + +// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. +func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { + return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil +} + +// GetUserStatus Get the online status of the user. +func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (*pbuser.GetUserStatusResp, error) { + res, err := s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.GetUserStatusResp{StatusList: res}, nil +} + +// SetUserStatus Synchronize user's online status. +func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (*pbuser.SetUserStatusResp, error) { + var ( + online []int32 + offline []int32 + ) + switch req.Status { + case constant.Online: + online = []int32{req.PlatformID} + case constant.Offline: + online = []int32{req.PlatformID} + } + if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { + return nil, err + } + return &pbuser.SetUserStatusResp{}, nil +} + +// GetSubscribeUsersStatus Get the online status of subscribers. +func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + return &pbuser.GetSubscribeUsersStatusResp{}, nil +} + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + for _, status := range req.Status { + if err := s.online.SetUserOnline(ctx, status.UserID, status.Online, status.Offline); err != nil { + return nil, err + } + } + return &pbuser.SetUserOnlineStatusResp{}, nil +} + +func (s *userServer) GetAllOnlineUsers(ctx context.Context, req *pbuser.GetAllOnlineUsersReq) (*pbuser.GetAllOnlineUsersResp, error) { + resMap, nextCursor, err := s.online.GetAllOnlineUsers(ctx, req.Cursor) + if err != nil { + return nil, err + } + resp := &pbuser.GetAllOnlineUsersResp{ + StatusList: make([]*pbuser.OnlineStatus, 0, len(resMap)), + NextCursor: nextCursor, + } + for userID, plats := range resMap { + resp.StatusList = append(resp.StatusList, &pbuser.OnlineStatus{ + UserID: userID, + Status: int32(datautil.If(len(plats) > 0, constant.Online, constant.Offline)), + PlatformIDs: plats, + }) + } + return resp, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index d0d3dbf603..8b22c8f9b7 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,17 +16,24 @@ package user import ( "context" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" + "errors" + "math/rand" + "strings" + "sync" + "time" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/protocol/group" + friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" - "math/rand" - "strings" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -46,8 +53,9 @@ import ( ) type userServer struct { + online cache.OnlineCache db controller.UserDatabase - friendNotificationSender *friend.FriendNotificationSender + friendNotificationSender *relation.FriendNotificationSender userNotificationSender *UserNotificationSender friendRpcClient *rpcclient.FriendRpcClient groupRpcClient *rpcclient.GroupRpcClient @@ -87,18 +95,18 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi return err } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) - userMongoDB := mgo.NewUserMongoDriver(mgocli.GetDB()) - database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx(), userMongoDB) + database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ + online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, friendRpcClient: &friendRpcClient, groupRpcClient: &groupRpcClient, - friendNotificationSender: friend.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, friend.WithDBFunc(database.FindWithError)), + friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, relation.WithDBFunc(database.FindWithError)), userNotificationSender: NewUserNotificationSender(config, &msgRpcClient, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), @@ -131,60 +139,62 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } + func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err } + if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } + + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } + data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - } + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } + return resp, nil } func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { @@ -297,6 +307,8 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR return nil, err } + prommetrics.UserRegisterCounter.Add(float64(len(users))) + s.webhookAfterUserRegister(ctx, &s.config.WebhooksConfig.AfterUserRegister, req) return resp, nil } @@ -318,76 +330,6 @@ func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDR return &pbuser.GetAllUserIDResp{Total: int32(total), UserIDs: userIDs}, nil } -// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. -func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (resp *pbuser.SubscribeOrCancelUsersStatusResp, err error) { - if req.Genre == constant.SubscriberUser { - err = s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err = s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } - return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil -} - -// GetUserStatus Get the online status of the user. -func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { - onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.GetUserStatusResp{StatusList: onlineStatusList}, nil -} - -// SetUserStatus Synchronize user's online status. -func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { - err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) - if err != nil { - return nil, err - } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - - return &pbuser.SetUserStatusResp{}, nil -} - -// GetSubscribeUsersStatus Get the online status of subscribers. -func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.db.GetUserStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil -} - // ProcessUserCommandAdd user general function add. func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.ProcessUserCommandAddReq) (*pbuser.ProcessUserCommandAddResp, error) { err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) @@ -683,3 +625,45 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } + +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} + +func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { + users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) + if err != nil { + return nil, err + } + return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil +} diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694b..dbb4e34f61 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -17,15 +17,18 @@ package tools import ( "context" "fmt" + "os" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "os" - "time" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -39,7 +42,7 @@ type CronTaskConfig struct { } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.RetainChatRecords) + log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } @@ -49,27 +52,85 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) - conn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) + + msgConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) if err != nil { return err } - cli := msg.NewMsgClient(conn) + + // thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + // if err != nil { + // return err + // } + + conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) + if err != nil { + return err + } + + msgClient := msg.NewMsgClient(msgConn) + conversationClient := pbconversation.NewConversationClient(conversationConn) + // thirdClient := third.NewThirdClient(thirdConn) + crontab := cron.New() - clearFunc := func() { + + // scheduled hard delete outdated Msgs in specific time. + clearMsgFunc := func() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - if _, err := cli.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { + log.ZDebug(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + + if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } - log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { return errs.Wrap(err) } - log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) + + // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. + msgDestructFunc := func() { + now := time.Now() + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) + log.ZDebug(ctx, "msg destruct cron start", "now", now) + + conversations, err := conversationClient.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) + if err != nil { + log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) + return + } else { + _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Destruct Msgs failed.", err) + return + } + } + log.ZDebug(ctx, "msg destruct cron task completed", "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, msgDestructFunc); err != nil { + return errs.Wrap(err) + } + + // // scheduled delete outdated file Objects and their datas in specific time. + // deleteObjectFunc := func() { + // now := time.Now() + // deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + // ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + // log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + // if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { + // log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + // return + // } + // log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + // } + // if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { + // return errs.Wrap(err) + // } + + log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index e79b477222..f4deb9fb1b 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -15,7 +15,7 @@ package apistruct import ( - sdkws "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/sdkws" ) // SendMsg defines the structure for sending messages with various metadata. @@ -55,6 +55,9 @@ type SendMsg struct { // OfflinePushInfo contains information for offline push notifications. OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + + // Ex stores extended fields + Ex string `json:"ex"` } // SendMsgReq extends SendMsg with the requirement of RecvID when SessionType indicates a one-on-one or notification chat. diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index f4a9f884c4..dc20b5104b 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -91,7 +91,7 @@ type OANotificationElem struct { NotificationType int32 `mapstructure:"notificationType" json:"notificationType" validate:"required"` Text string `mapstructure:"text" json:"text" validate:"required"` Url string `mapstructure:"url" json:"url"` - MixType int32 `mapstructure:"mixType" json:"mixType"` + MixType int32 `mapstructure:"mixType" json:"mixType" validate:"gte=0,lte=5"` PictureElem *PictureElem `mapstructure:"pictureElem" json:"pictureElem"` SoundElem *SoundElem `mapstructure:"soundElem" json:"soundElem"` VideoElem *VideoElem `mapstructure:"videoElem" json:"videoElem"` diff --git a/pkg/apistruct/msg_test.go b/pkg/apistruct/msg_test.go new file mode 100644 index 0000000000..28f878a9fd --- /dev/null +++ b/pkg/apistruct/msg_test.go @@ -0,0 +1 @@ +package apistruct diff --git a/pkg/callbackstruct/common.go b/pkg/callbackstruct/common.go index d6714f5f20..9d6a325a8d 100644 --- a/pkg/callbackstruct/common.go +++ b/pkg/callbackstruct/common.go @@ -35,6 +35,7 @@ type CommonCallbackReq struct { MsgFrom int32 `json:"msgFrom"` ContentType int32 `json:"contentType"` Status int32 `json:"status"` + SendTime int64 `json:"sendTime"` CreateTime int64 `json:"createTime"` Content string `json:"content"` Seq uint32 `json:"seq"` diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index 66e1598cda..89062ee0a0 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -18,7 +18,9 @@ const ( CallbackBeforeInviteJoinGroupCommand = "callbackBeforeInviteJoinGroupCommand" CallbackAfterJoinGroupCommand = "callbackAfterJoinGroupCommand" CallbackAfterSetGroupInfoCommand = "callbackAfterSetGroupInfoCommand" + CallbackAfterSetGroupInfoEXCommand = "callbackAfterSetGroupInfoCommandEX" CallbackBeforeSetGroupInfoCommand = "callbackBeforeSetGroupInfoCommand" + CallbackBeforeSetGroupInfoEXCommand = "callbackBeforeSetGroupInfoEXCommand" CallbackAfterRevokeMsgCommand = "callbackBeforeAfterMsgCommand" CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" @@ -56,7 +58,7 @@ const ( CallbackBeforeUpdateUserInfoCommand = "callbackBeforeUpdateUserInfoCommand" CallbackBeforeCreateGroupCommand = "callbackBeforeCreateGroupCommand" CallbackAfterCreateGroupCommand = "callbackAfterCreateGroupCommand" - CallbackBeforeMemberJoinGroupCommand = "callbackBeforeMemberJoinGroupCommand" + CallbackBeforeMembersJoinGroupCommand = "callbackBeforeMembersJoinGroupCommand" CallbackBeforeSetGroupMemberInfoCommand = "callbackBeforeSetGroupMemberInfoCommand" CallbackAfterSetGroupMemberInfoCommand = "callbackAfterSetGroupMemberInfoCommand" ) diff --git a/pkg/callbackstruct/group.go b/pkg/callbackstruct/group.go index e78d45ab4c..7fefa5b926 100644 --- a/pkg/callbackstruct/group.go +++ b/pkg/callbackstruct/group.go @@ -17,6 +17,7 @@ package callbackstruct import ( "github.com/openimsdk/open-im-server/v3/pkg/apistruct" common "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/wrapperspb" ) type CallbackCommand string @@ -59,16 +60,20 @@ type CallbackAfterCreateGroupResp struct { CommonCallbackResp } -type CallbackBeforeMemberJoinGroupReq struct { +type CallbackGroupMember struct { + UserID string `json:"userID"` + Ex string `json:"ex"` +} + +type CallbackBeforeMembersJoinGroupReq struct { CallbackCommand `json:"callbackCommand"` - GroupID string `json:"groupID"` - UserID string `json:"userID"` - Ex string `json:"ex"` - GroupEx string `json:"groupEx"` + GroupID string `json:"groupID"` + MembersList []*CallbackGroupMember `json:"memberList"` + GroupEx string `json:"groupEx"` } -type CallbackBeforeMemberJoinGroupResp struct { - CommonCallbackResp +type MemberJoinGroupCallBack struct { + UserID *string `json:"userID"` Nickname *string `json:"nickname"` FaceURL *string `json:"faceURL"` RoleLevel *int32 `json:"roleLevel"` @@ -76,6 +81,11 @@ type CallbackBeforeMemberJoinGroupResp struct { Ex *string `json:"ex"` } +type CallbackBeforeMembersJoinGroupResp struct { + CommonCallbackResp + MemberCallbackList []*MemberJoinGroupCallBack `json:"memberCallbackList"` +} + type CallbackBeforeSetGroupMemberInfoReq struct { CallbackCommand `json:"callbackCommand"` GroupID string `json:"groupID"` @@ -233,3 +243,48 @@ type CallbackAfterSetGroupInfoReq struct { type CallbackAfterSetGroupInfoResp struct { CommonCallbackResp } + +type CallbackBeforeSetGroupInfoEXReq struct { + CallbackCommand `json:"callbackCommand"` + OperationID string `json:"operationID"` + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackBeforeSetGroupInfoEXResp struct { + CommonCallbackResp + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackAfterSetGroupInfoEXReq struct { + CallbackCommand `json:"callbackCommand"` + OperationID string `json:"operationID"` + GroupID string `json:"groupID"` + GroupName string `json:"groupName"` + Notification *wrapperspb.StringValue `json:"notification"` + Introduction *wrapperspb.StringValue `json:"introduction"` + FaceURL *wrapperspb.StringValue `json:"faceURL"` + Ex *wrapperspb.StringValue `json:"ex"` + NeedVerification *wrapperspb.Int32Value `json:"needVerification"` + LookMemberInfo *wrapperspb.Int32Value `json:"lookMemberInfo"` + ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` +} + +type CallbackAfterSetGroupInfoEXResp struct { + CommonCallbackResp +} diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index ecdb0dd3ad..4088ecd09d 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/api" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -38,7 +39,7 @@ func NewApiCmd() *ApiCmd { DiscoveryConfigFilename: &apiConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 7d75a7da65..b35a95f395 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/auth" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -40,7 +41,7 @@ func NewAuthRpcCmd() *AuthRpcCmd { DiscoveryConfigFilename: &authConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 57ffa52bc4..bdb4447f48 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/conversation" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -43,7 +44,7 @@ func NewConversationRpcCmd() *ConversationRpcCmd { DiscoveryConfigFilename: &conversationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index fd44475246..d6c5e472e1 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/tools" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -38,7 +39,7 @@ func NewCronTaskCmd() *CronTaskCmd { DiscoveryConfigFilename: &cronTaskConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 8be1f77458..a564facd06 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -16,35 +16,36 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) type FriendRpcCmd struct { *RootCmd - ctx context.Context - configMap map[string]any - friendConfig *friend.Config + ctx context.Context + configMap map[string]any + relationConfig *relation.Config } func NewFriendRpcCmd() *FriendRpcCmd { - var friendConfig friend.Config - ret := &FriendRpcCmd{friendConfig: &friendConfig} + var relationConfig relation.Config + ret := &FriendRpcCmd{relationConfig: &relationConfig} ret.configMap = map[string]any{ - OpenIMRPCFriendCfgFileName: &friendConfig.RpcConfig, - RedisConfigFileName: &friendConfig.RedisConfig, - MongodbConfigFileName: &friendConfig.MongodbConfig, - ShareFileName: &friendConfig.Share, - NotificationFileName: &friendConfig.NotificationConfig, - WebhooksConfigFileName: &friendConfig.WebhooksConfig, - LocalCacheConfigFileName: &friendConfig.LocalCacheConfig, - DiscoveryConfigFilename: &friendConfig.Discovery, + OpenIMRPCFriendCfgFileName: &relationConfig.RpcConfig, + RedisConfigFileName: &relationConfig.RedisConfig, + MongodbConfigFileName: &relationConfig.MongodbConfig, + ShareFileName: &relationConfig.Share, + NotificationFileName: &relationConfig.NotificationConfig, + WebhooksConfigFileName: &relationConfig.WebhooksConfig, + LocalCacheConfigFileName: &relationConfig.LocalCacheConfig, + DiscoveryConfigFilename: &relationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } @@ -56,7 +57,7 @@ func (a *FriendRpcCmd) Exec() error { } func (a *FriendRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.friendConfig.Discovery, &a.friendConfig.RpcConfig.Prometheus, a.friendConfig.RpcConfig.RPC.ListenIP, - a.friendConfig.RpcConfig.RPC.RegisterIP, a.friendConfig.RpcConfig.RPC.Ports, - a.Index(), a.friendConfig.Share.RpcRegisterName.Friend, &a.friendConfig.Share, a.friendConfig, friend.Start) + return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, + a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.Ports, + a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f158b8c626..9b0fbf8de3 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -16,9 +16,11 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/group" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -44,7 +46,7 @@ func NewGroupRpcCmd() *GroupRpcCmd { DiscoveryConfigFilename: &groupConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } @@ -58,5 +60,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start) + a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index 91f7931fbe..bfd29398ef 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/msg" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewMsgRpcCmd() *MsgRpcCmd { DiscoveryConfigFilename: &msgConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 78004094c7..6363bfbf9e 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -16,9 +16,9 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/internal/msggateway" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -37,11 +37,12 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { ret.configMap = map[string]any{ OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, ShareFileName: &msgGatewayConfig.Share, + RedisConfigFileName: &msgGatewayConfig.RedisConfig, WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/msg_gateway_test.go b/pkg/common/cmd/msg_gateway_test.go index d820627b50..2b68a3e3ab 100644 --- a/pkg/common/cmd/msg_gateway_test.go +++ b/pkg/common/cmd/msg_gateway_test.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/utils/jsonutil" "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson/primitive" "math" "testing" ) @@ -59,3 +60,9 @@ func TestName(t *testing.T) { t.Logf("%+v\n", rReso) } + +func TestName1(t *testing.T) { + t.Log(primitive.NewObjectID().String()) + t.Log(primitive.NewObjectID().Hex()) + +} diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 0d48281e50..3643934135 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -16,8 +16,9 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -42,7 +43,7 @@ func NewMsgTransferCmd() *MsgTransferCmd { DiscoveryConfigFilename: &msgTransferConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 6e6014021e..ca22a697d2 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/push" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -36,7 +37,6 @@ func NewPushRpcCmd() *PushRpcCmd { ret.configMap = map[string]any{ OpenIMPushCfgFileName: &pushConfig.RpcConfig, RedisConfigFileName: &pushConfig.RedisConfig, - MongodbConfigFileName: &pushConfig.MongodbConfig, KafkaConfigFileName: &pushConfig.KafkaConfig, ShareFileName: &pushConfig.Share, NotificationFileName: &pushConfig.NotificationConfig, @@ -45,7 +45,7 @@ func NewPushRpcCmd() *PushRpcCmd { DiscoveryConfigFilename: &pushConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { ret.pushConfig.FcmConfigPath = ret.ConfigPath() return ret.runE() diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 08bb6d0642..5edea43773 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -19,6 +19,7 @@ import ( "path/filepath" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/spf13/cobra" @@ -128,22 +129,24 @@ func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { } func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { - err := log.InitFromConfig( + err := log.InitLoggerFromConfig( cmdOpts.loggerPrefixName, r.processName, + "", "", r.log.RemainLogLevel, r.log.IsStdout, r.log.IsJson, r.log.StorageLocation, r.log.RemainRotationCount, r.log.RotationTime, - config.Version, + version.Version, + r.log.IsSimplify, ) if err != nil { return errs.Wrap(err) } - return errs.Wrap(log.InitConsoleLogger(r.processName, r.log.RemainLogLevel, r.log.IsJson, config.Version)) + return errs.Wrap(log.InitConsoleLogger(r.processName, r.log.RemainLogLevel, r.log.IsJson, version.Version)) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index b6731f1ff4..a301b738fa 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/third" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -44,7 +45,7 @@ func NewThirdRpcCmd() *ThirdRpcCmd { DiscoveryConfigFilename: &thirdConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 674f9e3a6d..9a614afcab 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -16,9 +16,10 @@ package cmd import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/user" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -45,7 +46,7 @@ func NewUserRpcCmd() *UserRpcCmd { DiscoveryConfigFilename: &userConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) - ret.ctx = context.WithValue(context.Background(), "version", config.Version) + ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { return ret.runE() } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f5..59919208be 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,14 +15,16 @@ package config import ( + "strings" + "time" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "strings" - "time" ) type CacheConfig struct { @@ -47,6 +49,7 @@ type Log struct { RemainLogLevel int `mapstructure:"remainLogLevel"` IsStdout bool `mapstructure:"isStdout"` IsJson bool `mapstructure:"isJson"` + IsSimplify bool `mapstructure:"isSimplify"` WithStack bool `mapstructure:"withStack"` } @@ -70,18 +73,21 @@ type Mongo struct { MaxRetry int `mapstructure:"maxRetry"` } type Kafka struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ProducerAck string `mapstructure:"producerAck"` - CompressType string `mapstructure:"compressType"` - Address []string `mapstructure:"address"` - ToRedisTopic string `mapstructure:"toRedisTopic"` - ToMongoTopic string `mapstructure:"toMongoTopic"` - ToPushTopic string `mapstructure:"toPushTopic"` - ToRedisGroupID string `mapstructure:"toRedisGroupID"` - ToMongoGroupID string `mapstructure:"toMongoGroupID"` - ToPushGroupID string `mapstructure:"toPushGroupID"` - Tls TLSConfig `mapstructure:"tls"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + ProducerAck string `mapstructure:"producerAck"` + CompressType string `mapstructure:"compressType"` + Address []string `mapstructure:"address"` + ToRedisTopic string `mapstructure:"toRedisTopic"` + ToMongoTopic string `mapstructure:"toMongoTopic"` + ToPushTopic string `mapstructure:"toPushTopic"` + ToOfflinePushTopic string `mapstructure:"toOfflinePushTopic"` + ToRedisGroupID string `mapstructure:"toRedisGroupID"` + ToMongoGroupID string `mapstructure:"toMongoGroupID"` + ToPushGroupID string `mapstructure:"toPushGroupID"` + ToOfflineGroupID string `mapstructure:"toOfflinePushGroupID"` + + Tls TLSConfig `mapstructure:"tls"` } type TLSConfig struct { EnableTLS bool `mapstructure:"enableTLS"` @@ -94,8 +100,9 @@ type TLSConfig struct { type API struct { Api struct { - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + ListenIP string `mapstructure:"listenIP"` + Ports []int `mapstructure:"ports"` + CompressionLevel int `mapstructure:"compressionLevel"` } `mapstructure:"api"` Prometheus struct { Enable bool `mapstructure:"enable"` @@ -105,8 +112,9 @@ type API struct { } type CronTask struct { - ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` + CronExecuteTime string `mapstructure:"cronExecuteTime"` + RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { @@ -216,6 +224,7 @@ type Push struct { BadgeCount bool `mapstructure:"badgeCount"` Production bool `mapstructure:"production"` } `mapstructure:"iosPush"` + FullUserCache bool `mapstructure:"fullUserCache"` } type Auth struct { @@ -254,7 +263,8 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + EnableHistoryForNewMembers bool `mapstructure:"enableHistoryForNewMembers"` } type Msg struct { @@ -278,16 +288,8 @@ type Third struct { Enable string `mapstructure:"enable"` Cos Cos `mapstructure:"cos"` Oss Oss `mapstructure:"oss"` - Kodo struct { - Endpoint string `mapstructure:"endpoint"` - Bucket string `mapstructure:"bucket"` - BucketURL string `mapstructure:"bucketURL"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` - } `mapstructure:"kodo"` - Aws struct { + Kodo Kodo `mapstructure:"kodo"` + Aws struct { Endpoint string `mapstructure:"endpoint"` Region string `mapstructure:"region"` Bucket string `mapstructure:"bucket"` @@ -314,6 +316,16 @@ type Oss struct { PublicRead bool `mapstructure:"publicRead"` } +type Kodo struct { + Endpoint string `mapstructure:"endpoint"` + Bucket string `mapstructure:"bucket"` + BucketURL string `mapstructure:"bucketURL"` + AccessKeyID string `mapstructure:"accessKeyID"` + AccessKeySecret string `mapstructure:"accessKeySecret"` + SessionToken string `mapstructure:"sessionToken"` + PublicRead bool `mapstructure:"publicRead"` +} + type User struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` @@ -329,7 +341,8 @@ type Redis struct { Password string `mapstructure:"password"` ClusterMode bool `mapstructure:"clusterMode"` DB int `mapstructure:"storage"` - MaxRetry int `mapstructure:"MaxRetry"` + MaxRetry int `mapstructure:"maxRetry"` + PoolSize int `mapstructure:"poolSize"` } type BeforeConfig struct { @@ -339,8 +352,9 @@ type BeforeConfig struct { } type AfterConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` + Enable bool `mapstructure:"enable"` + Timeout int `mapstructure:"timeout"` + AttentionIds []string `mapstructure:"attentionIds"` } type Share struct { @@ -414,6 +428,8 @@ type Webhooks struct { BeforeInviteUserToGroup BeforeConfig `mapstructure:"beforeInviteUserToGroup"` AfterSetGroupInfo AfterConfig `mapstructure:"afterSetGroupInfo"` BeforeSetGroupInfo BeforeConfig `mapstructure:"beforeSetGroupInfo"` + AfterSetGroupInfoEX AfterConfig `mapstructure:"afterSetGroupInfoEX"` + BeforeSetGroupInfoEX BeforeConfig `mapstructure:"beforeSetGroupInfoEX"` AfterRevokeMsg AfterConfig `mapstructure:"afterRevokeMsg"` BeforeAddBlack BeforeConfig `mapstructure:"beforeAddBlack"` AfterAddFriend AfterConfig `mapstructure:"afterAddFriend"` @@ -464,6 +480,7 @@ func (r *Redis) Build() *redisutil.Config { Password: r.Password, DB: r.DB, MaxRetry: r.MaxRetry, + PoolSize: r.PoolSize, } } @@ -524,6 +541,18 @@ func (o *Oss) Build() *oss.Config { } } +func (o *Kodo) Build() *kodo.Config { + return &kodo.Config{ + Endpoint: o.Endpoint, + Bucket: o.Bucket, + BucketURL: o.BucketURL, + AccessKeyID: o.AccessKeyID, + AccessKeySecret: o.AccessKeySecret, + SessionToken: o.SessionToken, + PublicRead: o.PublicRead, + } +} + func (l *CacheConfig) Failed() time.Duration { return time.Second * time.Duration(l.FailedExpire) } diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index 256214565b..a0345fc7a5 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -36,3 +36,26 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) { //export IMENV_OPENIM_RPC_USER_RPC_PORTS="10110,10111,10112" assert.Equal(t, []int{10110, 10111, 10112}, user.RPC.Ports) } + +func TestLoadNotificationConfig(t *testing.T) { + var noti Notification + err := LoadConfig("../../../config/notification.yml", "IMENV_NOTIFICATION", ¬i) + assert.Nil(t, err) + assert.Equal(t, "Your friend's profile has been changed", noti.FriendRemarkSet.OfflinePush.Title) +} + +func TestLoadOpenIMThirdConfig(t *testing.T) { + var third Third + err := LoadConfig("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", &third) + assert.Nil(t, err) + assert.Equal(t, "enabled", third.Object.Enable) + assert.Equal(t, "https://oss-cn-chengdu.aliyuncs.com", third.Object.Oss.Endpoint) + assert.Equal(t, "my_bucket_name", third.Object.Oss.Bucket) + assert.Equal(t, "https://my_bucket_name.oss-cn-chengdu.aliyuncs.com", third.Object.Oss.BucketURL) + assert.Equal(t, "AKID1234567890", third.Object.Oss.AccessKeyID) + assert.Equal(t, "abc123xyz789", third.Object.Oss.AccessKeySecret) + assert.Equal(t, "session_token_value", third.Object.Oss.SessionToken) // Uncomment if session token is needed + assert.Equal(t, true, third.Object.Oss.PublicRead) + + // Environment: IMENV_OPENIM_RPC_THIRD_OBJECT_ENABLE=enabled;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ENDPOINT=https://oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKET=my_bucket_name;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKETURL=https://my_bucket_name.oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYID=AKID1234567890;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYSECRET=abc123xyz789;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_SESSIONTOKEN=session_token_value;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_PUBLICREAD=true +} diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index 28e9f5db60..08f82ac7d3 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -15,7 +15,6 @@ package config import ( - _ "embed" "os" "path/filepath" @@ -26,9 +25,6 @@ import ( "gopkg.in/yaml.v3" ) -//go:embed version -var Version string - const ( FileName = "config.yaml" NotificationFileName = "notification.yaml" diff --git a/pkg/common/config/version b/pkg/common/config/version deleted file mode 100644 index 240bba9069..0000000000 --- a/pkg/common/config/version +++ /dev/null @@ -1 +0,0 @@ -3.7.0 \ No newline at end of file diff --git a/pkg/common/convert/conversation.go b/pkg/common/convert/conversation.go index a76d7d9f6d..9389b02524 100644 --- a/pkg/common/convert/conversation.go +++ b/pkg/common/convert/conversation.go @@ -22,7 +22,7 @@ import ( func ConversationDB2Pb(conversationDB *model.Conversation) *conversation.Conversation { conversationPB := &conversation.Conversation{} - conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix() + conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.UnixMilli() if err := datautil.CopyStructFields(conversationPB, conversationDB); err != nil { return nil } @@ -35,7 +35,7 @@ func ConversationsDB2Pb(conversationsDB []*model.Conversation) (conversationsPB if err := datautil.CopyStructFields(conversationPB, conversationDB); err != nil { continue } - conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.Unix() + conversationPB.LatestMsgDestructTime = conversationDB.LatestMsgDestructTime.UnixMilli() conversationsPB = append(conversationsPB, conversationPB) } return conversationsPB diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index ccc574f51b..d824fa68e0 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -16,26 +16,26 @@ package convert import ( relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" "time" "github.com/openimsdk/protocol/sdkws" ) -func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { - result := make([]*sdkws.UserInfo, 0, len(users)) - for _, user := range users { - userPb := &sdkws.UserInfo{ - UserID: user.UserID, - Nickname: user.Nickname, - FaceURL: user.FaceURL, - Ex: user.Ex, - CreateTime: user.CreateTime.UnixMilli(), - AppMangerLevel: user.AppMangerLevel, - GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, - } - result = append(result, userPb) +func UserDB2Pb(user *relationtb.User) *sdkws.UserInfo { + return &sdkws.UserInfo{ + UserID: user.UserID, + Nickname: user.Nickname, + FaceURL: user.FaceURL, + Ex: user.Ex, + CreateTime: user.CreateTime.UnixMilli(), + AppMangerLevel: user.AppMangerLevel, + GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, } - return result +} + +func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { + return datautil.Slice(users, UserDB2Pb) } func UserPb2DB(user *sdkws.UserInfo) *relationtb.User { diff --git a/pkg/common/ginprometheus/ginprometheus.go b/pkg/common/ginprometheus/ginprometheus.go index c2e6bdcca0..64f8a0d8a4 100644 --- a/pkg/common/ginprometheus/ginprometheus.go +++ b/pkg/common/ginprometheus/ginprometheus.go @@ -14,430 +14,431 @@ package ginprometheus -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var defaultMetricPath = "/metrics" - -// counter, counter_vec, gauge, gauge_vec, -// histogram, histogram_vec, summary, summary_vec. -var ( - reqCounter = &Metric{ - ID: "reqCnt", - Name: "requests_total", - Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", - Type: "counter_vec", - Args: []string{"code", "method", "handler", "host", "url"}} - - reqDuration = &Metric{ - ID: "reqDur", - Name: "request_duration_seconds", - Description: "The HTTP request latencies in seconds.", - Type: "histogram_vec", - Args: []string{"code", "method", "url"}, - } - - resSize = &Metric{ - ID: "resSz", - Name: "response_size_bytes", - Description: "The HTTP response sizes in bytes.", - Type: "summary"} - - reqSize = &Metric{ - ID: "reqSz", - Name: "request_size_bytes", - Description: "The HTTP request sizes in bytes.", - Type: "summary"} - - standardMetrics = []*Metric{ - reqCounter, - reqDuration, - resSize, - reqSize, - } -) - -/* -RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control -the cardinality of the request counter's "url" label, which might be required in some contexts. -For instance, if for a "/customer/:name" route you don't want to generate a time series for every -possible customer name, you could use this function: - - func(c *gin.Context) string { - url := c.Request.URL.Path - for _, p := range c.Params { - if p.Key == "name" { - url = strings.Replace(url, p.Value, ":name", 1) - break - } - } - return url - } - -which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". -*/ -type RequestCounterURLLabelMappingFn func(c *gin.Context) string - -// Metric is a definition for the name, description, type, ID, and -// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. -type Metric struct { - MetricCollector prometheus.Collector - ID string - Name string - Description string - Type string - Args []string -} - -// Prometheus contains the metrics gathered by the instance and its path. -type Prometheus struct { - reqCnt *prometheus.CounterVec - reqDur *prometheus.HistogramVec - reqSz, resSz prometheus.Summary - router *gin.Engine - listenAddress string - Ppg PrometheusPushGateway - - MetricsList []*Metric - MetricsPath string - - ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn - - // gin.Context string to use as a prometheus URL label - URLLabelFromContext string -} - -// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). -type PrometheusPushGateway struct { - - // Push interval in seconds - PushIntervalSeconds time.Duration - - // Push Gateway URL in format http://domain:port - // where JOBNAME can be any string of your choice - PushGatewayURL string - - // Local metrics URL where metrics are fetched from, this could be omitted in the future - // if implemented using prometheus common/expfmt instead - MetricsURL string - - // pushgateway job name, defaults to "gin" - Job string -} - -// NewPrometheus generates a new set of metrics with a certain subsystem name. -func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { - if subsystem == "" { - subsystem = "app" - } - - var metricsList []*Metric - - if len(customMetricsList) > 1 { - panic("Too many args. NewPrometheus( string, ).") - } else if len(customMetricsList) == 1 { - metricsList = customMetricsList[0] - } - metricsList = append(metricsList, standardMetrics...) - - p := &Prometheus{ - MetricsList: metricsList, - MetricsPath: defaultMetricPath, - ReqCntURLLabelMappingFn: func(c *gin.Context) string { - return c.FullPath() // e.g. /user/:id , /user/:id/info - }, - } - - p.registerMetrics(subsystem) - - return p -} - -// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL -// every pushIntervalSeconds. Metrics are fetched from metricsURL. -func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { - p.Ppg.PushGatewayURL = pushGatewayURL - p.Ppg.MetricsURL = metricsURL - p.Ppg.PushIntervalSeconds = pushIntervalSeconds - p.startPushTicker() -} - -// SetPushGatewayJob job name, defaults to "gin". -func (p *Prometheus) SetPushGatewayJob(j string) { - p.Ppg.Job = j -} - -// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the -// same address of the gin engine that is being used. -func (p *Prometheus) SetListenAddress(address string) { - p.listenAddress = address - if p.listenAddress != "" { - p.router = gin.Default() - } -} - -// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of -// your content's access log). -func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { - p.listenAddress = listenAddress - if len(p.listenAddress) > 0 { - p.router = r - } -} - -// SetMetricsPath set metrics paths. -func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, prometheusHandler()) - return nil - } -} - -// SetMetricsPathWithAuth set metrics paths with authentication. -func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return nil - } - -} - -func (p *Prometheus) runServer() error { - return p.router.Run(p.listenAddress) -} - -func (p *Prometheus) getMetrics() []byte { - response, err := http.Get(p.Ppg.MetricsURL) - if err != nil { - return nil - } - - defer response.Body.Close() - - body, _ := io.ReadAll(response.Body) - return body -} - -var hostname, _ = os.Hostname() - -func (p *Prometheus) getPushGatewayURL() string { - if p.Ppg.Job == "" { - p.Ppg.Job = "gin" - } - return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname -} - -func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { - req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) - if err != nil { - return - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - fmt.Println("Error sending to push gateway error:", err.Error()) - } - - resp.Body.Close() -} - -func (p *Prometheus) startPushTicker() { - ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) - go func() { - for range ticker.C { - p.sendMetricsToPushGateway(p.getMetrics()) - } - }() -} - -// NewMetric associates prometheus.Collector based on Metric.Type. -func NewMetric(m *Metric, subsystem string) prometheus.Collector { - var metric prometheus.Collector - switch m.Type { - case "counter_vec": - metric = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "counter": - metric = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "gauge_vec": - metric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "gauge": - metric = prometheus.NewGauge( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "histogram_vec": - metric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "histogram": - metric = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "summary_vec": - metric = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "summary": - metric = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - } - return metric -} - -func (p *Prometheus) registerMetrics(subsystem string) { - for _, metricDef := range p.MetricsList { - metric := NewMetric(metricDef, subsystem) - if err := prometheus.Register(metric); err != nil { - fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) - } - - switch metricDef { - case reqCounter: - p.reqCnt = metric.(*prometheus.CounterVec) - case reqDuration: - p.reqDur = metric.(*prometheus.HistogramVec) - case resSize: - p.resSz = metric.(prometheus.Summary) - case reqSize: - p.reqSz = metric.(prometheus.Summary) - } - metricDef.MetricCollector = metric - } -} - -// Use adds the middleware to a gin engine. -func (p *Prometheus) Use(e *gin.Engine) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPath(e) -} - -// UseWithAuth adds the middleware to a gin engine with BasicAuth. -func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPathWithAuth(e, accounts) -} - -// HandlerFunc defines handler function for middleware. -func (p *Prometheus) HandlerFunc() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.URL.Path == p.MetricsPath { - c.Next() - return - } - - start := time.Now() - reqSz := computeApproximateRequestSize(c.Request) - - c.Next() - - status := strconv.Itoa(c.Writer.Status()) - elapsed := float64(time.Since(start)) / float64(time.Second) - resSz := float64(c.Writer.Size()) - - url := p.ReqCntURLLabelMappingFn(c) - if len(p.URLLabelFromContext) > 0 { - u, found := c.Get(p.URLLabelFromContext) - if !found { - u = "unknown" - } - url = u.(string) - } - p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) - p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() - p.reqSz.Observe(float64(reqSz)) - p.resSz.Observe(resSz) - } -} - -func prometheusHandler() gin.HandlerFunc { - h := promhttp.Handler() - return func(c *gin.Context) { - h.ServeHTTP(c.Writer, c.Request) - } -} - -func computeApproximateRequestSize(r *http.Request) int { - var s int - if r.URL != nil { - s = len(r.URL.Path) - } - - s += len(r.Method) - s += len(r.Proto) - for name, values := range r.Header { - s += len(name) - for _, value := range values { - s += len(value) - } - } - s += len(r.Host) - - // r.FormData and r.MultipartForm are assumed to be included in r.URL. - - if r.ContentLength != -1 { - s += int(r.ContentLength) - } - return s -} +// +//import ( +// "bytes" +// "fmt" +// "io" +// "net/http" +// "os" +// "strconv" +// "time" +// +// "github.com/gin-gonic/gin" +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" +//) +// +//var defaultMetricPath = "/metrics" +// +//// counter, counter_vec, gauge, gauge_vec, +//// histogram, histogram_vec, summary, summary_vec. +//var ( +// reqCounter = &Metric{ +// ID: "reqCnt", +// Name: "requests_total", +// Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", +// Type: "counter_vec", +// Args: []string{"code", "method", "handler", "host", "url"}} +// +// reqDuration = &Metric{ +// ID: "reqDur", +// Name: "request_duration_seconds", +// Description: "The HTTP request latencies in seconds.", +// Type: "histogram_vec", +// Args: []string{"code", "method", "url"}, +// } +// +// resSize = &Metric{ +// ID: "resSz", +// Name: "response_size_bytes", +// Description: "The HTTP response sizes in bytes.", +// Type: "summary"} +// +// reqSize = &Metric{ +// ID: "reqSz", +// Name: "request_size_bytes", +// Description: "The HTTP request sizes in bytes.", +// Type: "summary"} +// +// standardMetrics = []*Metric{ +// reqCounter, +// reqDuration, +// resSize, +// reqSize, +// } +//) +// +///* +//RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control +//the cardinality of the request counter's "url" label, which might be required in some contexts. +//For instance, if for a "/customer/:name" route you don't want to generate a time series for every +//possible customer name, you could use this function: +// +// func(c *gin.Context) string { +// url := c.Request.URL.Path +// for _, p := range c.Params { +// if p.Key == "name" { +// url = strings.Replace(url, p.Value, ":name", 1) +// break +// } +// } +// return url +// } +// +//which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". +//*/ +//type RequestCounterURLLabelMappingFn func(c *gin.Context) string +// +//// Metric is a definition for the name, description, type, ID, and +//// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. +//type Metric struct { +// MetricCollector prometheus.Collector +// ID string +// Name string +// Description string +// Type string +// Args []string +//} +// +//// Prometheus contains the metrics gathered by the instance and its path. +//type Prometheus struct { +// reqCnt *prometheus.CounterVec +// reqDur *prometheus.HistogramVec +// reqSz, resSz prometheus.Summary +// router *gin.Engine +// listenAddress string +// Ppg PrometheusPushGateway +// +// MetricsList []*Metric +// MetricsPath string +// +// ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn +// +// // gin.Context string to use as a prometheus URL label +// URLLabelFromContext string +//} +// +//// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). +//type PrometheusPushGateway struct { +// +// // Push interval in seconds +// PushIntervalSeconds time.Duration +// +// // Push Gateway URL in format http://domain:port +// // where JOBNAME can be any string of your choice +// PushGatewayURL string +// +// // Local metrics URL where metrics are fetched from, this could be omitted in the future +// // if implemented using prometheus common/expfmt instead +// MetricsURL string +// +// // pushgateway job name, defaults to "gin" +// Job string +//} +// +//// NewPrometheus generates a new set of metrics with a certain subsystem name. +//func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { +// if subsystem == "" { +// subsystem = "app" +// } +// +// var metricsList []*Metric +// +// if len(customMetricsList) > 1 { +// panic("Too many args. NewPrometheus( string, ).") +// } else if len(customMetricsList) == 1 { +// metricsList = customMetricsList[0] +// } +// metricsList = append(metricsList, standardMetrics...) +// +// p := &Prometheus{ +// MetricsList: metricsList, +// MetricsPath: defaultMetricPath, +// ReqCntURLLabelMappingFn: func(c *gin.Context) string { +// return c.FullPath() // e.g. /user/:id , /user/:id/info +// }, +// } +// +// p.registerMetrics(subsystem) +// +// return p +//} +// +//// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL +//// every pushIntervalSeconds. Metrics are fetched from metricsURL. +//func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { +// p.Ppg.PushGatewayURL = pushGatewayURL +// p.Ppg.MetricsURL = metricsURL +// p.Ppg.PushIntervalSeconds = pushIntervalSeconds +// p.startPushTicker() +//} +// +//// SetPushGatewayJob job name, defaults to "gin". +//func (p *Prometheus) SetPushGatewayJob(j string) { +// p.Ppg.Job = j +//} +// +//// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the +//// same address of the gin engine that is being used. +//func (p *Prometheus) SetListenAddress(address string) { +// p.listenAddress = address +// if p.listenAddress != "" { +// p.router = gin.Default() +// } +//} +// +//// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of +//// your content's access log). +//func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { +// p.listenAddress = listenAddress +// if len(p.listenAddress) > 0 { +// p.router = r +// } +//} +// +//// SetMetricsPath set metrics paths. +//func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, prometheusHandler()) +// return nil +// } +//} +// +//// SetMetricsPathWithAuth set metrics paths with authentication. +//func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return nil +// } +// +//} +// +//func (p *Prometheus) runServer() error { +// return p.router.Run(p.listenAddress) +//} +// +//func (p *Prometheus) getMetrics() []byte { +// response, err := http.Get(p.Ppg.MetricsURL) +// if err != nil { +// return nil +// } +// +// defer response.Body.Close() +// +// body, _ := io.ReadAll(response.Body) +// return body +//} +// +//var hostname, _ = os.Hostname() +// +//func (p *Prometheus) getPushGatewayURL() string { +// if p.Ppg.Job == "" { +// p.Ppg.Job = "gin" +// } +// return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname +//} +// +//func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { +// req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) +// if err != nil { +// return +// } +// +// client := &http.Client{} +// resp, err := client.Do(req) +// if err != nil { +// fmt.Println("Error sending to push gateway error:", err.Error()) +// } +// +// resp.Body.Close() +//} +// +//func (p *Prometheus) startPushTicker() { +// ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) +// go func() { +// for range ticker.C { +// p.sendMetricsToPushGateway(p.getMetrics()) +// } +// }() +//} +// +//// NewMetric associates prometheus.Collector based on Metric.Type. +//func NewMetric(m *Metric, subsystem string) prometheus.Collector { +// var metric prometheus.Collector +// switch m.Type { +// case "counter_vec": +// metric = prometheus.NewCounterVec( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "counter": +// metric = prometheus.NewCounter( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "gauge_vec": +// metric = prometheus.NewGaugeVec( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "gauge": +// metric = prometheus.NewGauge( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "histogram_vec": +// metric = prometheus.NewHistogramVec( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "histogram": +// metric = prometheus.NewHistogram( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "summary_vec": +// metric = prometheus.NewSummaryVec( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "summary": +// metric = prometheus.NewSummary( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// } +// return metric +//} +// +//func (p *Prometheus) registerMetrics(subsystem string) { +// for _, metricDef := range p.MetricsList { +// metric := NewMetric(metricDef, subsystem) +// if err := prometheus.Register(metric); err != nil { +// fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) +// } +// +// switch metricDef { +// case reqCounter: +// p.reqCnt = metric.(*prometheus.CounterVec) +// case reqDuration: +// p.reqDur = metric.(*prometheus.HistogramVec) +// case resSize: +// p.resSz = metric.(prometheus.Summary) +// case reqSize: +// p.reqSz = metric.(prometheus.Summary) +// } +// metricDef.MetricCollector = metric +// } +//} +// +//// Use adds the middleware to a gin engine. +//func (p *Prometheus) Use(e *gin.Engine) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPath(e) +//} +// +//// UseWithAuth adds the middleware to a gin engine with BasicAuth. +//func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPathWithAuth(e, accounts) +//} +// +//// HandlerFunc defines handler function for middleware. +//func (p *Prometheus) HandlerFunc() gin.HandlerFunc { +// return func(c *gin.Context) { +// if c.Request.URL.Path == p.MetricsPath { +// c.Next() +// return +// } +// +// start := time.Now() +// reqSz := computeApproximateRequestSize(c.Request) +// +// c.Next() +// +// status := strconv.Itoa(c.Writer.Status()) +// elapsed := float64(time.Since(start)) / float64(time.Second) +// resSz := float64(c.Writer.Size()) +// +// url := p.ReqCntURLLabelMappingFn(c) +// if len(p.URLLabelFromContext) > 0 { +// u, found := c.Get(p.URLLabelFromContext) +// if !found { +// u = "unknown" +// } +// url = u.(string) +// } +// p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) +// p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() +// p.reqSz.Observe(float64(reqSz)) +// p.resSz.Observe(resSz) +// } +//} +// +//func prometheusHandler() gin.HandlerFunc { +// h := promhttp.Handler() +// return func(c *gin.Context) { +// h.ServeHTTP(c.Writer, c.Request) +// } +//} +// +//func computeApproximateRequestSize(r *http.Request) int { +// var s int +// if r.URL != nil { +// s = len(r.URL.Path) +// } +// +// s += len(r.Method) +// s += len(r.Proto) +// for name, values := range r.Header { +// s += len(name) +// for _, value := range values { +// s += len(value) +// } +// } +// s += len(r.Host) +// +// // r.FormData and r.MultipartForm are assumed to be included in r.URL. +// +// if r.ContentLength != -1 { +// s += int(r.ContentLength) +// } +// return s +//} diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go new file mode 100644 index 0000000000..95b5c06b68 --- /dev/null +++ b/pkg/common/prommetrics/api.go @@ -0,0 +1,48 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "strconv" +) + +var ( + apiCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_count", + Help: "Total number of API calls", + }, + []string{"path", "method", "code"}, + ) + httpCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_count", + Help: "Total number of HTTP calls", + }, + []string{"path", "method", "status"}, + ) +) + +func ApiInit(prometheusPort int) error { + apiRegistry := prometheus.NewRegistry() + cs := append( + baseCollector, + apiCounter, + httpCounter, + ) + return Init(apiRegistry, prometheusPort, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) +} + +func APICall(path string, method string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "method": method, "code": strconv.Itoa(apiCode)}).Inc() +} + +func HttpCall(path string, method string, status int) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() +} + +//func ApiHandler() http.Handler { +// return promhttp.InstrumentMetricHandler( +// apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), +// ) +//} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go deleted file mode 100644 index 9f2e4c99d7..0000000000 --- a/pkg/common/prommetrics/gin_api.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prommetrics - -import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" - -/* -labels := prometheus.Labels{"label_one": "any", "label_two": "value"} -ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). -*/ -var ( - ApiCustomCnt = &ginprom.Metric{ - Name: "custom_total", - Description: "Custom counter events.", - Type: "counter_vec", - Args: []string{"label_one", "label_two"}, - } -) diff --git a/pkg/common/prommetrics/grpc_push.go b/pkg/common/prommetrics/grpc_push.go index 0b6c3e76f3..5c966310f7 100644 --- a/pkg/common/prommetrics/grpc_push.go +++ b/pkg/common/prommetrics/grpc_push.go @@ -23,4 +23,8 @@ var ( Name: "msg_offline_push_failed_total", Help: "The number of msg failed offline pushed", }) + MsgLoneTimePushCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "msg_long_time_push_total", + Help: "The number of messages with a push time exceeding 10 seconds", + }) ) diff --git a/pkg/common/prommetrics/grpc_user.go b/pkg/common/prommetrics/grpc_user.go new file mode 100644 index 0000000000..cc2fc42e61 --- /dev/null +++ b/pkg/common/prommetrics/grpc_user.go @@ -0,0 +1,10 @@ +package prommetrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + UserRegisterCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "user_register_total", + Help: "The number of user login", + }) +) diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 47e5d02b88..02e408d63b 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,44 +15,24 @@ package prommetrics import ( - gp "github.com/grpc-ecosystem/go-grpc-prometheus" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" + "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "net/http" ) -func NewGrpcPromObj(cusMetrics []prometheus.Collector) (*prometheus.Registry, *gp.ServerMetrics, error) { - reg := prometheus.NewRegistry() - grpcMetrics := gp.NewServerMetrics() - grpcMetrics.EnableHandlingTimeHistogram() - cusMetrics = append(cusMetrics, grpcMetrics, collectors.NewGoCollector()) - reg.MustRegister(cusMetrics...) - return reg, grpcMetrics, nil -} +const commonPath = "/metrics" -func GetGrpcCusMetrics(registerName string, share *config2.Share) []prometheus.Collector { - switch registerName { - case share.RpcRegisterName.MessageGateway: - return []prometheus.Collector{OnlineUserGauge} - case share.RpcRegisterName.Msg: - return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} - case "Transfer": - return []prometheus.Collector{MsgInsertRedisSuccessCounter, MsgInsertRedisFailedCounter, MsgInsertMongoSuccessCounter, MsgInsertMongoFailedCounter, SeqSetFailedCounter} - case share.RpcRegisterName.Push: - return []prometheus.Collector{MsgOfflinePushFailedCounter} - case share.RpcRegisterName.Auth: - return []prometheus.Collector{UserLoginCounter} - default: - return nil +var ( + baseCollector = []prometheus.Collector{ + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), } -} +) -func GetGinCusMetrics(name string) []*ginprometheus.Metric { - switch name { - case "Api": - return []*ginprometheus.Metric{ApiCustomCnt} - default: - return []*ginprometheus.Metric{ApiCustomCnt} - } +func Init(registry *prometheus.Registry, prometheusPort int, path string, handler http.Handler, cs ...prometheus.Collector) error { + registry.MustRegister(cs...) + srv := http.NewServeMux() + srv.Handle(path, handler) + return http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv) } diff --git a/pkg/common/prommetrics/prommetrics_test.go b/pkg/common/prommetrics/prommetrics_test.go index 65b05652f7..14b1aaff32 100644 --- a/pkg/common/prommetrics/prommetrics_test.go +++ b/pkg/common/prommetrics/prommetrics_test.go @@ -14,46 +14,39 @@ package prommetrics -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" -) - -func TestNewGrpcPromObj(t *testing.T) { - // Create a custom metric to pass into the NewGrpcPromObj function. - customMetric := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "test_metric", - Help: "This is a test metric.", - }) - cusMetrics := []prometheus.Collector{customMetric} - - // Call NewGrpcPromObj with the custom metrics. - reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) - - // Assert no error was returned. - assert.NoError(t, err) - - // Assert the registry was correctly initialized. - assert.NotNil(t, reg) - - // Assert the grpcMetrics was correctly initialized. - assert.NotNil(t, grpcMetrics) - - // Assert that the custom metric is registered. - mfs, err := reg.Gather() - assert.NoError(t, err) - assert.NotEmpty(t, mfs) // Ensure some metrics are present. - found := false - for _, mf := range mfs { - if *mf.Name == "test_metric" { - found = true - break - } - } - assert.True(t, found, "Custom metric not found in registry") -} +//func TestNewGrpcPromObj(t *testing.T) { +// // Create a custom metric to pass into the NewGrpcPromObj function. +// customMetric := prometheus.NewCounter(prometheus.CounterOpts{ +// Name: "test_metric", +// Help: "This is a test metric.", +// }) +// cusMetrics := []prometheus.Collector{customMetric} +// +// // Call NewGrpcPromObj with the custom metrics. +// reg, grpcMetrics, err := NewGrpcPromObj(cusMetrics) +// +// // Assert no error was returned. +// assert.NoError(t, err) +// +// // Assert the registry was correctly initialized. +// assert.NotNil(t, reg) +// +// // Assert the grpcMetrics was correctly initialized. +// assert.NotNil(t, grpcMetrics) +// +// // Assert that the custom metric is registered. +// mfs, err := reg.Gather() +// assert.NoError(t, err) +// assert.NotEmpty(t, mfs) // Ensure some metrics are present. +// found := false +// for _, mf := range mfs { +// if *mf.Name == "test_metric" { +// found = true +// break +// } +// } +// assert.True(t, found, "Custom metric not found in registry") +//} //func TestGetGrpcCusMetrics(t *testing.T) { // conf := config2.NewGlobalConfig() diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go new file mode 100644 index 0000000000..7162fa7e80 --- /dev/null +++ b/pkg/common/prommetrics/rpc.go @@ -0,0 +1,68 @@ +package prommetrics + +import ( + gp "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "strconv" +) + +const rpcPath = commonPath + +var ( + grpcMetrics *gp.ServerMetrics + rpcCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_count", + Help: "Total number of RPC calls", + }, + []string{"name", "path", "code"}, + ) +) + +func RpcInit(cs []prometheus.Collector, prometheusPort int) error { + reg := prometheus.NewRegistry() + cs = append(append( + baseCollector, + rpcCounter, + ), cs...) + return Init(reg, prometheusPort, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) +} + +func RPCCall(name string, path string, code int) { + rpcCounter.With(prometheus.Labels{"name": name, "path": path, "code": strconv.Itoa(code)}).Inc() +} + +func GetGrpcServerMetrics() *gp.ServerMetrics { + if grpcMetrics == nil { + grpcMetrics = gp.NewServerMetrics() + grpcMetrics.EnableHandlingTimeHistogram() + } + return grpcMetrics +} + +func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Collector { + switch registerName { + case share.RpcRegisterName.MessageGateway: + return []prometheus.Collector{OnlineUserGauge} + case share.RpcRegisterName.Msg: + return []prometheus.Collector{ + SingleChatMsgProcessSuccessCounter, + SingleChatMsgProcessFailedCounter, + GroupChatMsgProcessSuccessCounter, + GroupChatMsgProcessFailedCounter, + } + case share.RpcRegisterName.Push: + return []prometheus.Collector{ + MsgOfflinePushFailedCounter, + MsgLoneTimePushCounter, + } + case share.RpcRegisterName.Auth: + return []prometheus.Collector{UserLoginCounter} + case share.RpcRegisterName.User: + return []prometheus.Collector{UserRegisterCounter} + default: + return nil + } +} diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index 197b6f7fc8..f0abb82856 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -16,6 +16,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -40,3 +41,16 @@ var ( Help: "The number of failed set seq", }) ) + +func TransferInit(prometheusPort int) error { + reg := prometheus.NewRegistry() + cs := append( + baseCollector, + MsgInsertRedisSuccessCounter, + MsgInsertRedisFailedCounter, + MsgInsertMongoSuccessCounter, + MsgInsertMongoFailedCounter, + SeqSetFailedCounter, + ) + return Init(reg, prometheusPort, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 069c92012a..85a6c3d512 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -17,9 +17,9 @@ package startrpc import ( "context" "fmt" - config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" - "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" "net" "net/http" "os" @@ -29,7 +29,6 @@ import ( "syscall" "time" - grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -38,14 +37,13 @@ import ( "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/network" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, - registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, + registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) @@ -56,15 +54,11 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort, "prometheusPorts", prometheusConfig.Ports) rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) + listener, err := net.Listen( "tcp", rpcTcpAddr, ) - if err != nil { - return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } - - defer listener.Close() client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { return err @@ -77,13 +71,18 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC return err } - var reg *prometheus.Registry - var metric *grpcprometheus.ServerMetrics + //var reg *prometheus.Registry + //var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append( + options, mw.GrpcServer(), + prommetricsUnaryInterceptor(rpcRegisterName), + prommetricsStreamInterceptor(rpcRegisterName), + ) } else { options = append(options, mw.GrpcServer()) } @@ -122,13 +121,18 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC netDone <- struct{}{} return } - metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } + //metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + //} }() } @@ -158,7 +162,6 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC } return nil case <-netDone: - close(netDone) return netErr } } @@ -176,3 +179,25 @@ func gracefulStopWithCtx(ctx context.Context, f func()) error { return nil } } + +func prommetricsUnaryInterceptor(rpcRegisterName string) grpc.ServerOption { + getCode := func(err error) int { + if err == nil { + return 0 + } + rpcErr, ok := err.(interface{ GRPCStatus() *status.Status }) + if !ok { + return -1 + } + return int(rpcErr.GRPCStatus().Code()) + } + return grpc.ChainUnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + resp, err := handler(ctx, req) + prommetrics.RPCCall(rpcRegisterName, info.FullMethod, getCode(err)) + return resp, err + }) +} + +func prommetricsStreamInterceptor(rpcRegisterName string) grpc.ServerOption { + return grpc.ChainStreamInterceptor() +} diff --git a/pkg/common/storage/cache/cachekey/conversation.go b/pkg/common/storage/cache/cachekey/conversation.go index aea4ceec68..d19fcc5767 100644 --- a/pkg/common/storage/cache/cachekey/conversation.go +++ b/pkg/common/storage/cache/cachekey/conversation.go @@ -23,6 +23,7 @@ const ( SuperGroupRecvMsgNotNotifyUserIDsKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:" SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:" ConversationNotReceiveMessageUserIDsKey = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:" + ConversationUserMaxKey = "CONVERSATION_USER_MAX:" ) func GetConversationKey(ownerUserID, conversationID string) string { @@ -56,3 +57,7 @@ func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string { func GetUserConversationIDsHashKey(ownerUserID string) string { return ConversationIDsHashKey + ownerUserID } + +func GetConversationUserMaxVersionKey(userID string) string { + return ConversationUserMaxKey + userID +} diff --git a/pkg/common/storage/cache/cachekey/friend.go b/pkg/common/storage/cache/cachekey/friend.go index 9691b1f5c9..8a053ca324 100644 --- a/pkg/common/storage/cache/cachekey/friend.go +++ b/pkg/common/storage/cache/cachekey/friend.go @@ -19,6 +19,8 @@ const ( TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" FriendKey = "FRIEND_INFO:" IsFriendKey = "IS_FRIEND:" // local cache key + //FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendMaxVersionKey = "FRIEND_MAX_VERSION:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -33,6 +35,14 @@ func GetFriendKey(ownerUserID, friendUserID string) string { return FriendKey + ownerUserID + "-" + friendUserID } +func GetFriendMaxVersionKey(ownerUserID string) string { + return FriendMaxVersionKey + ownerUserID +} + func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } + +//func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { +// return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +//} diff --git a/pkg/common/storage/cache/cachekey/group.go b/pkg/common/storage/cache/cachekey/group.go index 681121ecba..2ef42c0ff4 100644 --- a/pkg/common/storage/cache/cachekey/group.go +++ b/pkg/common/storage/cache/cachekey/group.go @@ -28,6 +28,8 @@ const ( JoinedGroupsKey = "JOIN_GROUPS_KEY:" GroupMemberNumKey = "GROUP_MEMBER_NUM_CACHE:" GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:" + GroupMemberMaxVersionKey = "GROUP_MEMBER_MAX_VERSION:" + GroupJoinMaxVersionKey = "GROUP_JOIN_MAX_VERSION:" ) func GetGroupInfoKey(groupID string) string { @@ -57,3 +59,11 @@ func GetGroupMemberNumKey(groupID string) string { func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string { return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel)) } + +func GetGroupMemberMaxVersionKey(groupID string) string { + return GroupMemberMaxVersionKey + groupID +} + +func GetJoinGroupMaxVersionKey(userID string) string { + return GroupJoinMaxVersionKey + userID +} diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go new file mode 100644 index 0000000000..40f09cb5ae --- /dev/null +++ b/pkg/common/storage/cache/cachekey/online.go @@ -0,0 +1,20 @@ +package cachekey + +import ( + "strings" + "time" +) + +const ( + OnlineKey = "ONLINE:" + OnlineChannel = "online_change" + OnlineExpire = time.Hour / 2 +) + +func GetOnlineKey(userID string) string { + return OnlineKey + userID +} + +func GetOnlineKeyUserID(key string) string { + return strings.TrimPrefix(key, OnlineKey) +} diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index 3f0ce98a4d..b32e78300c 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -1,38 +1,30 @@ -// Copyright Β© 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cachekey const ( - maxSeq = "MAX_SEQ:" - minSeq = "MIN_SEQ:" - conversationUserMinSeq = "CON_USER_MIN_SEQ:" - hasReadSeq = "HAS_READ_SEQ:" + MallocSeq = "MALLOC_SEQ:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" + + SeqUserMaxSeq = "SEQ_USER_MAX:" + SeqUserMinSeq = "SEQ_USER_MIN:" + SeqUserReadSeq = "SEQ_USER_READ:" ) -func GetMaxSeqKey(conversationID string) string { - return maxSeq + conversationID +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID } -func GetMinSeqKey(conversationID string) string { - return minSeq + conversationID +func GetSeqUserMaxSeqKey(conversationID string, userID string) string { + return SeqUserMaxSeq + conversationID + ":" + userID } -func GetHasReadSeqKey(conversationID string, userID string) string { - return hasReadSeq + userID + ":" + conversationID +func GetSeqUserMinSeqKey(conversationID string, userID string) string { + return SeqUserMinSeq + conversationID + ":" + userID } -func GetConversationUserMinSeqKey(conversationID, userID string) string { - return conversationUserMinSeq + conversationID + "u:" + userID +func GetSeqUserReadSeqKey(conversationID string, userID string) string { + return SeqUserReadSeq + conversationID + ":" + userID } diff --git a/pkg/common/storage/cache/cachekey/user.go b/pkg/common/storage/cache/cachekey/user.go index 7d06d4f751..473ca1b12f 100644 --- a/pkg/common/storage/cache/cachekey/user.go +++ b/pkg/common/storage/cache/cachekey/user.go @@ -17,7 +17,6 @@ package cachekey const ( UserInfoKey = "USER_INFO:" UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:" - olineStatusKey = "ONLINE_STATUS:" ) func GetUserInfoKey(userID string) string { @@ -27,7 +26,3 @@ func GetUserInfoKey(userID string) string { func GetUserGlobalRecvMsgOptKey(userID string) string { return UserGlobalRecvMsgOptKey + userID } - -func GetOnlineStatusKey(modKey string) string { - return olineStatusKey + modKey -} diff --git a/pkg/common/storage/cache/conversation.go b/pkg/common/storage/cache/conversation.go index f34fd599f8..bc17614836 100644 --- a/pkg/common/storage/cache/conversation.go +++ b/pkg/common/storage/cache/conversation.go @@ -54,4 +54,8 @@ type ConversationCache interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache + + DelConversationVersionUserIDs(userIDs ...string) ConversationCache + + FindMaxConversationUserVersion(ctx context.Context, userID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index acff829f86..b451d36757 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -32,4 +32,16 @@ type FriendCache interface { DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + + DelMaxFriendVersion(ownerUserIDs ...string) FriendCache + + //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + + FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 53e2cd1c74..1ec0462956 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -36,7 +36,6 @@ type GroupCache interface { DelGroupMembersHash(groupID string) GroupCache GetGroupMemberIDs(ctx context.Context, groupID string) (groupMemberIDs []string, err error) - GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (groupMemberIDs map[string][]string, err error) DelGroupMemberIDs(groupID string) GroupCache @@ -46,7 +45,6 @@ type GroupCache interface { GetGroupMemberInfo(ctx context.Context, groupID, userID string) (groupMember *model.GroupMember, err error) GetGroupMembersInfo(ctx context.Context, groupID string, userID []string) (groupMembers []*model.GroupMember, err error) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) - GetGroupMembersPage(ctx context.Context, groupID string, userID []string, showNumber, pageNumber int32) (total uint32, groupMembers []*model.GroupMember, err error) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) @@ -59,4 +57,13 @@ type GroupCache interface { GetGroupRolesLevelMemberInfo(ctx context.Context, groupID string, roleLevels []int32) ([]*model.GroupMember, error) GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache + + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + DelMaxGroupMemberVersion(groupIDs ...string) GroupCache + DelMaxJoinGroupVersion(userIDs ...string) GroupCache + FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) + BatchFindMaxGroupMemberVersion(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) + FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) } diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go new file mode 100644 index 0000000000..d21ae616a6 --- /dev/null +++ b/pkg/common/storage/cache/online.go @@ -0,0 +1,9 @@ +package cache + +import "context" + +type OnlineCache interface { + GetOnline(ctx context.Context, userID string) ([]int32, error) + SetUserOnline(ctx context.Context, userID string, online, offline []int32) error + GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) +} diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go new file mode 100644 index 0000000000..1810ac9939 --- /dev/null +++ b/pkg/common/storage/cache/redis/batch.go @@ -0,0 +1,98 @@ +package redis + +import ( + "context" + "encoding/json" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + "time" + "unsafe" +) + +func getRocksCacheRedisClient(cli *rockscache.Client) redis.UniversalClient { + type Client struct { + rdb redis.UniversalClient + _ rockscache.Options + _ singleflight.Group + } + return (*Client)(unsafe.Pointer(cli)).rdb +} + +func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v *V) K, fn func(ctx context.Context, ids []K) ([]*V, error)) ([]*V, error) { + if len(ids) == 0 { + return nil, nil + } + findKeys := make([]string, 0, len(ids)) + keyId := make(map[string]K) + for _, id := range ids { + key := idKey(id) + if _, ok := keyId[key]; ok { + continue + } + keyId[key] = id + findKeys = append(findKeys, key) + } + slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), findKeys) + if err != nil { + return nil, err + } + result := make([]*V, 0, len(findKeys)) + for _, keys := range slotKeys { + indexCache, err := rcClient.FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { + queryIds := make([]K, 0, len(idx)) + idIndex := make(map[K]int) + for _, index := range idx { + id := keyId[keys[index]] + idIndex[id] = index + queryIds = append(queryIds, id) + } + values, err := fn(ctx, queryIds) + if err != nil { + log.ZError(ctx, "batchGetCache query database failed", err, "keys", keys, "queryIds", queryIds) + return nil, err + } + if len(values) == 0 { + return map[int]string{}, nil + } + cacheIndex := make(map[int]string) + for _, value := range values { + id := vId(value) + index, ok := idIndex[id] + if !ok { + continue + } + bs, err := json.Marshal(value) + if err != nil { + log.ZError(ctx, "marshal failed", err) + return nil, err + } + cacheIndex[index] = string(bs) + } + return cacheIndex, nil + }) + if err != nil { + return nil, errs.WrapMsg(err, "FetchBatch2 failed") + } + for index, data := range indexCache { + if data == "" { + continue + } + var value V + if err := json.Unmarshal([]byte(data), &value); err != nil { + return nil, errs.WrapMsg(err, "Unmarshal failed") + } + if cb, ok := any(&value).(BatchCacheCallback[K]); ok { + cb.BatchCache(keyId[keys[index]]) + } + result = append(result, &value) + } + } + return result, nil +} + +type BatchCacheCallback[K comparable] interface { + BatchCache(id K) +} diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 95f6699047..420ebdf777 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -23,12 +23,15 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw/specialerror" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" ) +const ( + rocksCacheTimeout = 11 * time.Second +) + // BatchDeleterRedis is a concrete implementation of the BatchDeleter interface based on Redis and RocksCache. type BatchDeleterRedis struct { redisClient redis.UniversalClient @@ -107,6 +110,8 @@ func (c *BatchDeleterRedis) AddKeys(keys ...string) { // GetRocksCacheOptions returns the default configuration options for RocksCache. func GetRocksCacheOptions() *rockscache.Options { opts := rockscache.NewDefaultOptions() + opts.LockExpire = rocksCacheTimeout + opts.WaitReplicasTimeout = rocksCacheTimeout opts.StrongConsistency = true opts.RandomExpireAdjustment = 0.2 @@ -119,6 +124,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { + //log.ZError(ctx, "getCache query database failed", err, "key", key) return "", err } bs, err := json.Marshal(t) @@ -147,30 +153,30 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin return t, nil } -func batchGetCache[T any, K comparable]( - ctx context.Context, - rcClient *rockscache.Client, - expire time.Duration, - keys []K, - keyFn func(key K) string, - fns func(ctx context.Context, key K) (T, error), -) ([]T, error) { - if len(keys) == 0 { - return nil, nil - } - res := make([]T, 0, len(keys)) - for _, key := range keys { - val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { - return fns(ctx, key) - }) - if err != nil { - if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { - continue - } - return nil, errs.Wrap(err) - } - res = append(res, val) - } - - return res, nil -} +//func batchGetCache[T any, K comparable]( +// ctx context.Context, +// rcClient *rockscache.Client, +// expire time.Duration, +// keys []K, +// keyFn func(key K) string, +// fns func(ctx context.Context, key K) (T, error), +//) ([]T, error) { +// if len(keys) == 0 { +// return nil, nil +// } +// res := make([]T, 0, len(keys)) +// for _, key := range keys { +// val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { +// return fns(ctx, key) +// }) +// if err != nil { +// if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { +// continue +// } +// return nil, errs.Wrap(err) +// } +// res = append(res, val) +// } +// +// return res, nil +//} diff --git a/pkg/common/storage/cache/redis/batch_test.go b/pkg/common/storage/cache/redis/batch_test.go new file mode 100644 index 0000000000..bbb6d76f15 --- /dev/null +++ b/pkg/common/storage/cache/redis/batch_test.go @@ -0,0 +1,55 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "testing" +) + +func TestName(t *testing.T) { + //var rocks rockscache.Client + //rdb := getRocksCacheRedisClient(&rocks) + //t.Log(rdb == nil) + + ctx := context.Background() + rdb, err := redisutil.NewRedisClient(ctx, (&config.Redis{ + Address: []string{"172.16.8.48:16379"}, + Password: "openIM123", + DB: 3, + }).Build()) + if err != nil { + panic(err) + } + mgocli, err := mongoutil.NewMongoDB(ctx, (&config.Mongo{ + Address: []string{"172.16.8.48:37017"}, + Database: "openim_v3", + Username: "openIM", + Password: "openIM123", + MaxPoolSize: 100, + MaxRetry: 1, + }).Build()) + if err != nil { + panic(err) + } + //userMgo, err := mgo.NewUserMongo(mgocli.GetDB()) + //if err != nil { + // panic(err) + //} + //rock := rockscache.NewClient(rdb, rockscache.NewDefaultOptions()) + mgoSeqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + panic(err) + } + seqUser := NewSeqUserCacheRedis(rdb, mgoSeqUser) + + res, err := seqUser.GetUserReadSeqs(ctx, "2110910952", []string{"sg_2920732023", "sg_345762580"}) + if err != nil { + panic(err) + } + + t.Log(res) + +} diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 8c0393dd56..95e680afb4 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -95,6 +95,10 @@ func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID strin return cachekey.GetUserConversationIDsHashKey(ownerUserID) } +func (c *ConversationRedisCache) getConversationUserMaxVersionKey(ownerUserID string) string { + return cachekey.GetConversationUserMaxVersionKey(ownerUserID) +} + func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { return getCache(ctx, c.rcClient, c.getConversationIDsKey(ownerUserID), c.expireTime, func(ctx context.Context) ([]string, error) { return c.conversationDB.FindUserIDAllConversationID(ctx, ownerUserID) @@ -160,10 +164,12 @@ func (c *ConversationRedisCache) DelConversations(ownerUserID string, conversati } func (c *ConversationRedisCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*model.Conversation, error) { - return batchGetCache(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { + return batchGetCache2(ctx, c.rcClient, c.expireTime, conversationIDs, func(conversationID string) string { return c.getConversationKey(ownerUserID, conversationID) - }, func(ctx context.Context, conversationID string) (*model.Conversation, error) { - return c.conversationDB.Take(ctx, ownerUserID, conversationID) + }, func(conversation *model.Conversation) string { + return conversation.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) { + return c.conversationDB.Find(ctx, ownerUserID, conversationIDs) }) } @@ -233,6 +239,19 @@ func (c *ConversationRedisCache) DelConversationNotReceiveMessageUserIDs(convers for _, conversationID := range conversationIDs { cache.AddKeys(c.getConversationNotReceiveMessageUserIDsKey(conversationID)) } + return cache +} +func (c *ConversationRedisCache) DelConversationVersionUserIDs(userIDs ...string) cache.ConversationCache { + cache := c.CloneConversationCache() + for _, userID := range userIDs { + cache.AddKeys(c.getConversationUserMaxVersionKey(userID)) + } return cache } + +func (c *ConversationRedisCache) FindMaxConversationUserVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, c.rcClient, c.getConversationUserMaxVersionKey(userID), c.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index f76e5ff6ba..be4687794f 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -16,6 +16,8 @@ package redis import ( "context" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -25,7 +27,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -38,6 +39,7 @@ type FriendCacheRedis struct { friendDB database.Friend expireTime time.Duration rcClient *rockscache.Client + syncCount int } // NewFriendCacheRedis creates a new instance of FriendCacheRedis. @@ -68,6 +70,10 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } +func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { + return cachekey.GetFriendMaxVersionKey(ownerUserID) +} + // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string { return cachekey.GetTwoWayFriendsIDsKey(ownerUserID) @@ -151,3 +157,30 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } + +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendKey(ownerUserID, friendUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + +func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendMaxVersionKey(ownerUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + +func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) + }) +} diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 2de03906f1..736111df31 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -17,6 +17,8 @@ package redis import ( "context" "fmt" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -27,9 +29,7 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -111,34 +111,20 @@ func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLeve return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel) } -func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { - key := g.getGroupInfoKey(group.GroupID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } - - return 0, errIndex +func (g *GroupCacheRedis) getGroupMemberMaxVersionKey(groupID string) string { + return cachekey.GetGroupMemberMaxVersionKey(groupID) } -func (g *GroupCacheRedis) GetGroupMemberIndex(groupMember *model.GroupMember, keys []string) (int, error) { - key := g.getGroupMemberInfoKey(groupMember.GroupID, groupMember.UserID) - for i, _key := range keys { - if _key == key { - return i, nil - } - } +func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { + return cachekey.GetJoinGroupMaxVersionKey(userID) +} - return 0, errIndex +func (g *GroupCacheRedis) getGroupID(group *model.Group) string { + return group.GroupID } func (g *GroupCacheRedis) GetGroupsInfo(ctx context.Context, groupIDs []string) (groups []*model.Group, err error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { - return g.getGroupInfoKey(groupID) - }, func(ctx context.Context, groupID string) (*model.Group, error) { - return g.groupDB.Take(ctx, groupID) - }) + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, g.getGroupInfoKey, g.getGroupID, g.groupDB.Find) } func (g *GroupCacheRedis) GetGroupInfo(ctx context.Context, groupID string) (group *model.Group, err error) { @@ -226,19 +212,6 @@ func (g *GroupCacheRedis) GetGroupMemberIDs(ctx context.Context, groupID string) }) } -func (g *GroupCacheRedis) GetGroupsMemberIDs(ctx context.Context, groupIDs []string) (map[string][]string, error) { - m := make(map[string][]string) - for _, groupID := range groupIDs { - userIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - m[groupID] = userIDs - } - - return m, nil -} - func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { cache := g.CloneGroupCache() cache.AddKeys(g.getGroupMemberIDsKey(groupID)) @@ -246,9 +219,17 @@ func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { return cache } +func (g *GroupCacheRedis) findUserJoinedGroupID(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + if err != nil { + return nil, err + } + return g.groupDB.FindJoinSortGroupID(ctx, groupIDs) +} + func (g *GroupCacheRedis) GetJoinedGroupIDs(ctx context.Context, userID string) (joinedGroupIDs []string, err error) { return getCache(ctx, g.rcClient, g.getJoinedGroupsKey(userID), g.expireTime, func(ctx context.Context) ([]string, error) { - return g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + return g.findUserJoinedGroupID(ctx, userID) }) } @@ -270,33 +251,15 @@ func (g *GroupCacheRedis) GetGroupMemberInfo(ctx context.Context, groupID, userI } func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { - return batchGetCache(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, userIDs, func(userID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, userID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.UserID + }, func(ctx context.Context, userIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.Find(ctx, groupID, userIDs) }) } -func (g *GroupCacheRedis) GetGroupMembersPage( - ctx context.Context, - groupID string, - userIDs []string, - showNumber, pageNumber int32, -) (total uint32, groupMembers []*model.GroupMember, err error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return 0, nil, err - } - if userIDs != nil { - userIDs = datautil.BothExist(userIDs, groupMemberIDs) - } else { - userIDs = groupMemberIDs - } - groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber))) - - return uint32(len(userIDs)), groupMembers, err -} - func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) { groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) if err != nil { @@ -306,14 +269,6 @@ func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID st return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) } -func (g *GroupCacheRedis) GetAllGroupMemberInfo(ctx context.Context, groupID string) ([]*model.GroupMember, error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - return g.GetGroupMembersInfo(ctx, groupID, groupMemberIDs) -} - func (g *GroupCacheRedis) DelGroupMembersInfo(groupID string, userIDs ...string) cache.GroupCache { keys := make([]string, 0, len(userIDs)) for _, userID := range userIDs { @@ -393,16 +348,66 @@ func (g *GroupCacheRedis) GetGroupRolesLevelMemberInfo(ctx context.Context, grou return g.GetGroupMembersInfo(ctx, groupID, userIDs) } -func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) (_ []*model.GroupMember, err error) { +func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) { if len(groupIDs) == 0 { + var err error groupIDs, err = g.GetJoinedGroupIDs(ctx, userID) if err != nil { return nil, err } } - return batchGetCache(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, func(groupID string) string { return g.getGroupMemberInfoKey(groupID, userID) - }, func(ctx context.Context, groupID string) (*model.GroupMember, error) { - return g.groupMemberDB.Take(ctx, groupID, userID) + }, func(member *model.GroupMember) string { + return member.GroupID + }, func(ctx context.Context, groupIDs []string) ([]*model.GroupMember, error) { + return g.groupMemberDB.FindInGroup(ctx, userID, groupIDs) + }) +} + +func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(groupIDs)) + for _, groupID := range groupIDs { + keys = append(keys, g.getGroupMemberMaxVersionKey(groupID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + keys = append(keys, g.getJoinGroupMaxVersionKey(userID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, 0, 0) + }) +} + +func (g *GroupCacheRedis) BatchFindMaxGroupMemberVersion(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) { + return batchGetCache2(ctx, g.rcClient, g.expireTime, groupIDs, + func(groupID string) string { + return g.getGroupMemberMaxVersionKey(groupID) + }, func(versionLog *model.VersionLog) string { + return versionLog.DID + }, func(ctx context.Context, groupIDs []string) ([]*model.VersionLog, error) { + // create two slices with len is groupIDs, just need 0 + versions := make([]uint, len(groupIDs)) + limits := make([]int, len(groupIDs)) + + return g.groupMemberDB.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) + }) +} + +func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0) }) } diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 2d21cfe135..30f367bb75 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -183,5 +183,4 @@ func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, return nil, nil, err } return seqMsgs, failedSeqs, nil - } diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go new file mode 100644 index 0000000000..b6c90264e1 --- /dev/null +++ b/pkg/common/storage/cache/redis/online.go @@ -0,0 +1,137 @@ +package redis + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "strconv" + "strings" + "time" +) + +func NewUserOnline(rdb redis.UniversalClient) cache.OnlineCache { + return &userOnline{ + rdb: rdb, + expire: cachekey.OnlineExpire, + channelName: cachekey.OnlineChannel, + } +} + +type userOnline struct { + rdb redis.UniversalClient + expire time.Duration + channelName string +} + +func (s *userOnline) getUserOnlineKey(userID string) string { + return cachekey.GetOnlineKey(userID) +} + +func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, error) { + members, err := s.rdb.ZRangeByScore(ctx, s.getUserOnlineKey(userID), &redis.ZRangeBy{ + Min: strconv.FormatInt(time.Now().Unix(), 10), + Max: "+inf", + }).Result() + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs := make([]int32, 0, len(members)) + for _, member := range members { + val, err := strconv.Atoi(member) + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs = append(platformIDs, int32(val)) + } + return platformIDs, nil +} + +func (s *userOnline) GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) { + result := make(map[string][]int32) + + keys, nextCursor, err := s.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", cachekey.OnlineKey), constant.ParamMaxLength).Result() + if err != nil { + return nil, 0, err + } + + for _, key := range keys { + userID := cachekey.GetOnlineKeyUserID(key) + strValues, err := s.rdb.ZRange(ctx, key, 0, -1).Result() + if err != nil { + return nil, 0, err + } + + values := make([]int32, 0, len(strValues)) + for _, value := range strValues { + intValue, err := strconv.Atoi(value) + if err != nil { + return nil, 0, errs.Wrap(err) + } + values = append(values, int32(intValue)) + } + + result[userID] = values + } + + return result, nextCursor, nil +} + +func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { + script := ` + local key = KEYS[1] + local score = ARGV[3] + local num1 = redis.call("ZCARD", key) + redis.call("ZREMRANGEBYSCORE", key, "-inf", ARGV[2]) + for i = 5, tonumber(ARGV[4])+4 do + redis.call("ZREM", key, ARGV[i]) + end + local num2 = redis.call("ZCARD", key) + for i = 5+tonumber(ARGV[4]), #ARGV do + redis.call("ZADD", key, score, ARGV[i]) + end + redis.call("EXPIRE", key, ARGV[1]) + local num3 = redis.call("ZCARD", key) + local change = (num1 ~= num2) or (num2 ~= num3) + if change then + local members = redis.call("ZRANGE", key, 0, -1) + table.insert(members, "1") + return members + else + return {"0"} + end +` + now := time.Now() + argv := make([]any, 0, 2+len(online)+len(offline)) + argv = append(argv, int32(s.expire/time.Second), now.Unix(), now.Add(s.expire).Unix(), int32(len(offline))) + for _, platformID := range offline { + argv = append(argv, platformID) + } + for _, platformID := range online { + argv = append(argv, platformID) + } + keys := []string{s.getUserOnlineKey(userID)} + platformIDs, err := s.rdb.Eval(ctx, script, keys, argv).StringSlice() + if err != nil { + log.ZError(ctx, "redis SetUserOnline", err, "userID", userID, "online", online, "offline", offline) + return err + } + if len(platformIDs) == 0 { + return errs.ErrInternalServer.WrapMsg("SetUserOnline redis lua invalid return value") + } + if platformIDs[len(platformIDs)-1] != "0" { + log.ZDebug(ctx, "redis SetUserOnline push", "userID", userID, "online", online, "offline", offline, "platformIDs", platformIDs[:len(platformIDs)-1]) + platformIDs[len(platformIDs)-1] = userID + msg := strings.Join(platformIDs, ":") + if err := s.rdb.Publish(ctx, s.channelName, msg).Err(); err != nil { + return errs.Wrap(err) + } + } else { + log.ZDebug(ctx, "redis SetUserOnline not push", "userID", userID, "online", online, "offline", offline) + } + return nil +} diff --git a/pkg/common/storage/cache/redis/online_test.go b/pkg/common/storage/cache/redis/online_test.go new file mode 100644 index 0000000000..0306f6f5d7 --- /dev/null +++ b/pkg/common/storage/cache/redis/online_test.go @@ -0,0 +1,51 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/redisutil" + "testing" + "time" +) + +/* +address: [ 172.16.8.48:7001, 172.16.8.48:7002, 172.16.8.48:7003, 172.16.8.48:7004, 172.16.8.48:7005, 172.16.8.48:7006 ] +username: +password: passwd123 +clusterMode: true +db: 0 +maxRetry: 10 +*/ +func TestName111111(t *testing.T) { + conf := config.Redis{ + Address: []string{ + "172.16.8.124:7001", + "172.16.8.124:7002", + "172.16.8.124:7003", + "172.16.8.124:7004", + "172.16.8.124:7005", + "172.16.8.124:7006", + }, + ClusterMode: true, + Password: "passwd123", + //Address: []string{"localhost:16379"}, + //Password: "openIM123", + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*1000) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, conf.Build()) + if err != nil { + panic(err) + } + online := NewUserOnline(rdb) + + userID := "a123456" + t.Log(online.GetOnline(ctx, userID)) + t.Log(online.SetUserOnline(ctx, userID, []int32{1, 2, 3, 4}, nil)) + t.Log(online.GetOnline(ctx, userID)) + +} + +func TestName111(t *testing.T) { + +} diff --git a/pkg/common/storage/cache/redis/redis_shard_manager.go b/pkg/common/storage/cache/redis/redis_shard_manager.go index 98d70dabf9..17e5fecf6f 100644 --- a/pkg/common/storage/cache/redis/redis_shard_manager.go +++ b/pkg/common/storage/cache/redis/redis_shard_manager.go @@ -2,6 +2,7 @@ package redis import ( "context" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -109,7 +110,7 @@ func (rsm *RedisShardManager) ProcessKeysBySlot( func groupKeysBySlot(ctx context.Context, redisClient redis.UniversalClient, keys []string) (map[int64][]string, error) { slots := make(map[int64][]string) clusterClient, isCluster := redisClient.(*redis.ClusterClient) - if isCluster { + if isCluster && len(keys) > 1 { pipe := clusterClient.Pipeline() cmds := make([]*redis.IntCmd, len(keys)) for i, key := range keys { @@ -195,3 +196,16 @@ func ProcessKeysBySlot( } return nil } + +func DeleteCacheBySlot(ctx context.Context, rcClient *rockscache.Client, keys []string) error { + switch len(keys) { + case 0: + return nil + case 1: + return rcClient.TagAsDeletedBatch2(ctx, keys) + default: + return ProcessKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), keys, func(ctx context.Context, slot int64, keys []string) error { + return rcClient.TagAsDeletedBatch2(ctx, keys) + }) + } +} diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index 76dd921a50..0000000000 --- a/pkg/common/storage/cache/redis/seq.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redis - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" - "github.com/redis/go-redis/v9" -) - -func NewSeqCache(rdb redis.UniversalClient) cache.SeqCache { - return &seqCache{rdb: rdb} -} - -type seqCache struct { - rdb redis.UniversalClient -} - -func (c *seqCache) getMaxSeqKey(conversationID string) string { - return cachekey.GetMaxSeqKey(conversationID) -} - -func (c *seqCache) getMinSeqKey(conversationID string) string { - return cachekey.GetMinSeqKey(conversationID) -} - -func (c *seqCache) getHasReadSeqKey(conversationID string, userID string) string { - return cachekey.GetHasReadSeqKey(conversationID, userID) -} - -func (c *seqCache) getConversationUserMinSeqKey(conversationID, userID string) string { - return cachekey.GetConversationUserMinSeqKey(conversationID, userID) -} - -func (c *seqCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error { - return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err()) -} - -func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey func(conversationID string) string) (int64, error) { - val, err := c.rdb.Get(ctx, getkey(conversationID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) { - m = make(map[string]int64, len(items)) - for i, v := range items { - res, err := c.rdb.Get(ctx, getkey(v)).Result() - if err != nil && err != redis.Nil { - return nil, errs.Wrap(err) - } - val := stringutil.StringToInt64(res) - if val != 0 { - m[items[i]] = val - } - } - - return m, nil -} - -func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -} - -func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -} - -func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { - for conversationID, seq := range seqs { - if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { - return errs.Wrap(err) - } - } - return nil -} - -func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return c.setSeqs(ctx, seqs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMinSeqKey) -} - -func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, userIDs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getConversationUserMinSeqKey(conversationID, userID), minSeq, 0).Err()) -} - -func (c *seqCache) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(conversationID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getHasReadSeqKey(conversationID, userID), hasReadSeq, 0).Err()) -} - -func (c *seqCache) SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(userID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getHasReadSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, err - } - return val, nil -} diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go new file mode 100644 index 0000000000..7fe849193e --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -0,0 +1,333 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "time" +) + +func NewSeqConversationCacheRedis(rdb redis.UniversalClient, mgo database.SeqConversation) cache.SeqConversationCache { + return &seqConversationCacheRedis{ + rdb: rdb, + mgo: mgo, + lockTime: time.Second * 3, + dataTime: time.Hour * 24 * 365, + minSeqExpireTime: time.Hour, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqConversationCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqConversation + rocks *rockscache.Client + lockTime time.Duration + dataTime time.Duration + minSeqExpireTime time.Duration +} + +func (s *seqConversationCacheRedis) getMinSeqKey(conversationID string) string { + return cachekey.GetMallocMinSeqKey(conversationID) +} + +func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return s.SetMinSeqs(ctx, map[string]int64{conversationID: seq}) +} + +func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache(ctx, s.rocks, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) +} + +func (s *seqConversationCacheRedis) getSingleMaxSeq(ctx context.Context, conversationID string) (map[string]int64, error) { + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return nil, err + } + return map[string]int64{conversationID: seq}, nil +} + +func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []string, keyConversationID map[string]string, seqs map[string]int64) error { + result := make([]*redis.StringCmd, len(keys)) + pipe := s.rdb.Pipeline() + for i, key := range keys { + result[i] = pipe.HGet(ctx, key, "CURR") + } + if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + return errs.Wrap(err) + } + var notFoundKey []string + for i, r := range result { + req, err := r.Int64() + if err == nil { + seqs[keyConversationID[keys[i]]] = req + } else if errors.Is(err, redis.Nil) { + notFoundKey = append(notFoundKey, keys[i]) + } else { + return errs.Wrap(err) + } + } + for _, key := range notFoundKey { + conversationID := keyConversationID[key] + seq, err := s.GetMaxSeq(ctx, conversationID) + if err != nil { + return err + } + seqs[conversationID] = seq + } + return nil +} + +func (s *seqConversationCacheRedis) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { + switch len(conversationIDs) { + case 0: + return map[string]int64{}, nil + case 1: + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + keys := make([]string, 0, len(conversationIDs)) + keyConversationID := make(map[string]string, len(conversationIDs)) + for _, conversationID := range conversationIDs { + key := s.getSeqMallocKey(conversationID) + if _, ok := keyConversationID[key]; ok { + continue + } + keys = append(keys, key) + keyConversationID[key] = conversationID + } + if len(keys) == 1 { + return s.getSingleMaxSeq(ctx, conversationIDs[0]) + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return nil, err + } + seqs := make(map[string]int64, len(conversationIDs)) + for _, keys := range slotKeys { + if err := s.batchGetMaxSeq(ctx, keys, keyConversationID, seqs); err != nil { + return nil, err + } + } + return seqs, nil +} + +func (s *seqConversationCacheRedis) getSeqMallocKey(conversationID string) string { + return cachekey.GetMallocSeqKey(conversationID) +} + +func (s *seqConversationCacheRedis) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { + if lastSeq < currSeq { + return 0, errs.New("lastSeq must be greater than currSeq") + } + // 0: success + // 1: success the lock has expired, but has not been locked by anyone else + // 2: already locked, but not by yourself + script := ` +local key = KEYS[1] +local lockValue = ARGV[1] +local dataSecond = ARGV[2] +local curr_seq = tonumber(ARGV[3]) +local last_seq = tonumber(ARGV[4]) +if redis.call("EXISTS", key) == 0 then + redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) + redis.call("EXPIRE", key, dataSecond) + return 1 +end +if redis.call("HGET", key, "LOCK") ~= lockValue then + return 2 +end +redis.call("HDEL", key, "LOCK") +redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) +redis.call("EXPIRE", key, dataSecond) +return 0 +` + result, err := s.rdb.Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq).Int64() + if err != nil { + return 0, errs.Wrap(err) + } + return result, nil +} + +// malloc size=0 is to get the current seq size>0 is to allocate seq +func (s *seqConversationCacheRedis) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0: success + // 1: need to obtain and lock + // 2: already locked + // 3: exceeded the maximum value and locked + script := ` +local key = KEYS[1] +local size = tonumber(ARGV[1]) +local lockSecond = ARGV[2] +local dataSecond = ARGV[3] +local result = {} +if redis.call("EXISTS", key) == 0 then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 1) + table.insert(result, lockValue) + return result +end +if redis.call("HEXISTS", key, "LOCK") == 1 then + table.insert(result, 2) + return result +end +local curr_seq = tonumber(redis.call("HGET", key, "CURR")) +local last_seq = tonumber(redis.call("HGET", key, "LAST")) +if size == 0 then + redis.call("EXPIRE", key, dataSecond) + table.insert(result, 0) + table.insert(result, curr_seq) + table.insert(result, last_seq) + return result +end +local max_seq = curr_seq + size +if max_seq > last_seq then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("HSET", key, "CURR", last_seq) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 3) + table.insert(result, curr_seq) + table.insert(result, last_seq) + table.insert(result, lockValue) + return result +end +redis.call("HSET", key, "CURR", max_seq) +redis.call("EXPIRE", key, dataSecond) +table.insert(result, 0) +table.insert(result, curr_seq) +table.insert(result, last_seq) +return result +` + result, err := s.rdb.Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second)).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return result, nil +} + +func (s *seqConversationCacheRedis) wait(ctx context.Context) error { + timer := time.NewTimer(time.Second / 4) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *seqConversationCacheRedis) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { + for i := 0; i < 10; i++ { + state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) + if err != nil { + log.ZError(ctx, "set seq cache failed", err, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq, "count", i+1) + if err := s.wait(ctx); err != nil { + return + } + continue + } + switch state { + case 0: // ideal state + case 1: + log.ZWarn(ctx, "set seq cache lock not found", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + case 2: + log.ZWarn(ctx, "set seq cache lock to be held by someone else", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + default: + log.ZError(ctx, "set seq cache lock unknown state", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + } + return + } + log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) +} + +func (s *seqConversationCacheRedis) getMallocSize(conversationID string, size int64) int64 { + if size == 0 { + return 0 + } + var basicSize int64 + if msgprocessor.IsGroupConversationID(conversationID) { + basicSize = 100 + } else { + basicSize = 50 + } + basicSize += size + return basicSize +} + +func (s *seqConversationCacheRedis) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errs.New("size must be greater than 0") + } + key := s.getSeqMallocKey(conversationID) + for i := 0; i < 10; i++ { + states, err := s.malloc(ctx, key, size) + if err != nil { + return 0, err + } + switch states[0] { + case 0: // success + return states[1], nil + case 1: // not found + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) + return seq, nil + case 2: // locked + if err := s.wait(ctx); err != nil { + return 0, err + } + continue + case 3: // exceeded cache max value + currSeq := states[1] + lastSeq := states[2] + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + if lastSeq == seq { + s.setSeqRetry(ctx, key, states[3], currSeq+size, seq+mallocSize) + return currSeq, nil + } else { + log.ZWarn(ctx, "malloc seq not equal cache last seq", nil, "conversationID", conversationID, "currSeq", currSeq, "lastSeq", lastSeq, "mallocSeq", seq) + s.setSeqRetry(ctx, key, states[3], seq+size, seq+mallocSize) + return seq, nil + } + default: + log.ZError(ctx, "malloc seq unknown state", nil, "state", states[0], "conversationID", conversationID, "size", size) + return 0, errs.New(fmt.Sprintf("unknown state: %d", states[0])) + } + } + log.ZError(ctx, "malloc seq retrying still failed", nil, "conversationID", conversationID, "size", size) + return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) +} + +func (s *seqConversationCacheRedis) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + return s.Malloc(ctx, conversationID, 0) +} + +func (s *seqConversationCacheRedis) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + keys = append(keys, s.getMinSeqKey(conversationID)) + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} diff --git a/pkg/common/storage/cache/redis/seq_conversation_test.go b/pkg/common/storage/cache/redis/seq_conversation_test.go new file mode 100644 index 0000000000..1a40624b8c --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation_test.go @@ -0,0 +1,109 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +func newTestSeq() *seqConversationCacheRedis { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqConversationMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return NewSeqConversationCacheRedis(rdb, model).(*seqConversationCacheRedis) +} + +func TestSeq(t *testing.T) { + ts := newTestSeq() + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 128 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 10 + cID := strconv.Itoa(index * 1) + for i := 1; ; i++ { + //first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) + } + } +} + +func TestDel(t *testing.T) { + ts := newTestSeq() + for i := 1; i < 100; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "100", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second) + } +} + +func TestSeqMalloc(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "100")) +} + +func TestMinSeq(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMinSeq(context.Background(), "10000000")) +} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go new file mode 100644 index 0000000000..0cedfeee12 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -0,0 +1,183 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +func NewSeqUserCacheRedis(rdb redis.UniversalClient, mgo database.SeqUser) cache.SeqUser { + return &seqUserCacheRedis{ + rdb: rdb, + mgo: mgo, + readSeqWriteRatio: 100, + expireTime: time.Hour * 24 * 7, + readExpireTime: time.Hour * 24 * 30, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqUserCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqUser + rocks *rockscache.Client + expireTime time.Duration + readExpireTime time.Duration + readSeqWriteRatio int64 +} + +func (s *seqUserCacheRedis) getSeqUserMaxSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMaxSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserMinSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMinSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserReadSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetUserMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetUserMaxSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetUserMinSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.SetUserMinSeqs(ctx, userID, map[string]int64{conversationID: seq}) +} + +func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetUserReadSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) + if err != nil { + return err + } + if dbSeq < seq { + if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + return errs.Wrap(err) + } + } + return nil +} + +func (s *seqUserCacheRedis) SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.mgo.SetUserReadSeq(ctx, conversationID, userID, seq) +} + +func (s *seqUserCacheRedis) SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + for conversationID, seq := range seqs { + if err := s.mgo.SetUserMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + keys = append(keys, s.getSeqUserMinSeqKey(conversationID, userID)) + } + return DeleteCacheBySlot(ctx, s.rocks, keys) +} + +func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + keys := make([]string, 0, len(seqs)) + keySeq := make(map[string]int64) + for conversationID, seq := range seqs { + key := s.getSeqUserReadSeqKey(conversationID, userID) + keys = append(keys, key) + keySeq[key] = seq + } + slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + if err != nil { + return err + } + for _, keys := range slotKeys { + pipe := s.rdb.Pipeline() + for _, key := range keys { + pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10)) + pipe.Expire(ctx, key, s.readExpireTime) + } + if _, err := pipe.Exec(ctx); err != nil { + return err + } + } + return nil +} + +func (s *seqUserCacheRedis) SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error { + if len(seqs) == 0 { + return nil + } + if err := s.setUserRedisReadSeqs(ctx, userID, seqs); err != nil { + return err + } + return nil +} + +func (s *seqUserCacheRedis) GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { + res, err := batchGetCache2(ctx, s.rocks, s.readExpireTime, conversationIDs, func(conversationID string) string { + return s.getSeqUserReadSeqKey(conversationID, userID) + }, func(v *readSeqModel) string { + return v.ConversationID + }, func(ctx context.Context, conversationIDs []string) ([]*readSeqModel, error) { + seqs, err := s.mgo.GetUserReadSeqs(ctx, userID, conversationIDs) + if err != nil { + return nil, err + } + res := make([]*readSeqModel, 0, len(seqs)) + for conversationID, seq := range seqs { + res = append(res, &readSeqModel{ConversationID: conversationID, Seq: seq}) + } + return res, nil + }) + if err != nil { + return nil, err + } + data := make(map[string]int64) + for _, v := range res { + data[v.ConversationID] = v.Seq + } + return data, nil +} + +var _ BatchCacheCallback[string] = (*readSeqModel)(nil) + +type readSeqModel struct { + ConversationID string + Seq int64 +} + +func (r *readSeqModel) BatchCache(conversationID string) { + r.ConversationID = conversationID +} + +func (r *readSeqModel) UnmarshalJSON(bytes []byte) (err error) { + r.Seq, err = strconv.ParseInt(string(bytes), 10, 64) + return +} + +func (r *readSeqModel) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatInt(r.Seq, 10)), nil +} diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go new file mode 100644 index 0000000000..0059c81db9 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -0,0 +1,111 @@ +package redis + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + mgo2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "log" + "strconv" + "sync/atomic" + "testing" + "time" +) + +func newTestOnline() *userOnline { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 0, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &userOnline{rdb: rdb, expire: time.Hour, channelName: "user_online"} +} + +func TestOnline(t *testing.T) { + ts := newTestOnline() + var count atomic.Int64 + for i := 0; i < 64; i++ { + go func(userID string) { + var err error + for i := 0; ; i++ { + if i%2 == 0 { + err = ts.SetUserOnline(context.Background(), userID, []int32{5, 6}, []int32{7, 8, 9}) + } else { + err = ts.SetUserOnline(context.Background(), userID, []int32{1, 2, 3}, []int32{4, 5, 6}) + } + if err != nil { + panic(err) + } + count.Add(1) + } + }(strconv.Itoa(10000 + i)) + } + + ticker := time.NewTicker(time.Second) + for range ticker.C { + t.Log(count.Swap(0)) + } +} + +func TestGetOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pIDs, err := ts.GetOnline(ctx, "10000") + if err != nil { + panic(err) + } + t.Log(pIDs) +} + +func TestRecvOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) + + _, err := pubsub.Receive(ctx) + if err != nil { + log.Fatalf("Could not subscribe: %v", err) + } + + ch := pubsub.Channel() + + for msg := range ch { + fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) + } +} + +func TestName1(t *testing.T) { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 0, + } + rdb := redis.NewClient(opt) + + mgo, err := mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo2.NewSeqUserMongo(mgo.Database("openim_v3")) + if err != nil { + panic(err) + } + seq := NewSeqUserCacheRedis(rdb, model) + + res, err := seq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"}) + if err != nil { + panic(err) + } + t.Log(res) + +} diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index 3de01563bf..f6b4907302 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -16,21 +16,14 @@ package redis import ( "context" - "encoding/json" - "errors" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "hash/crc32" - "strconv" "time" ) @@ -61,8 +54,8 @@ func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, } } -func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { - return cachekey.GetOnlineStatusKey(modKey) +func (u *UserCacheRedis) getUserID(user *model.User) string { + return user.UserID } func (u *UserCacheRedis) CloneUserCache() cache.UserCache { @@ -90,11 +83,7 @@ func (u *UserCacheRedis) GetUserInfo(ctx context.Context, userID string) (userIn } func (u *UserCacheRedis) GetUsersInfo(ctx context.Context, userIDs []string) ([]*model.User, error) { - return batchGetCache(ctx, u.rcClient, u.expireTime, userIDs, func(userID string) string { - return u.getUserInfoKey(userID) - }, func(ctx context.Context, userID string) (*model.User, error) { - return u.userDB.Take(ctx, userID) - }) + return batchGetCache2(ctx, u.rcClient, u.expireTime, userIDs, u.getUserInfoKey, u.getUserID, u.userDB.Find) } func (u *UserCacheRedis) DelUsersInfo(userIDs ...string) cache.UserCache { @@ -130,174 +119,3 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } - -// GetUserStatus get user status. -func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) - for _, userID := range userIDs { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - var onlineStatus user.OnlineStatus - key := u.getOnlineStatusKey(modKey) - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - // key or field does not exist - userStatus = append(userStatus, &user.OnlineStatus{ - UserID: userID, - Status: constant.Offline, - PlatformIDs: nil, - }) - - continue - } else { - return nil, errs.Wrap(err) - } - } - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return nil, errs.Wrap(err) - } - onlineStatus.UserID = userID - onlineStatus.Status = constant.Online - userStatus = append(userStatus, &onlineStatus) - } - - return userStatus, nil -} - -// SetUserStatus Set the user status and save it in redis. -func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - key := u.getOnlineStatusKey(modKey) - log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) - isNewKey, err := u.rdb.Exists(ctx, key).Result() - if err != nil { - return errs.Wrap(err) - } - if isNewKey == 0 { - if status == constant.Online { - onlineStatus := user.OnlineStatus{ - UserID: userID, - Status: constant.Online, - PlatformIDs: []int32{platformID}, - } - jsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - u.rdb.Expire(ctx, key, userOlineStatusExpireTime) - - return nil - } - } - - isNil := false - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - isNil = true - } else { - return errs.Wrap(err) - } - } - - if status == constant.Offline { - err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) - if err != nil { - return err - } - } else { - err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} - -func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { - if isNil { - log.ZWarn(ctx, "this user not online,maybe trigger order not right", - err, "userStatus", status) - - return nil - } - var onlineStatus user.OnlineStatus - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - var newPlatformIDs []int32 - for _, val := range onlineStatus.PlatformIDs { - if val != platformID { - newPlatformIDs = append(newPlatformIDs, val) - } - } - if newPlatformIDs == nil { - _, err = u.rdb.HDel(ctx, key, userID).Result() - if err != nil { - return errs.Wrap(err) - } - } else { - onlineStatus.PlatformIDs = newPlatformIDs - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} - -func (u *UserCacheRedis) refreshStatusOnline(ctx context.Context, userID string, platformID int32, isNil bool, err error, result, key string) error { - var onlineStatus user.OnlineStatus - if !isNil { - err := json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return errs.Wrap(err) - } - onlineStatus.PlatformIDs = RemoveRepeatedElementsInList(append(onlineStatus.PlatformIDs, platformID)) - } else { - onlineStatus.PlatformIDs = append(onlineStatus.PlatformIDs, platformID) - } - onlineStatus.Status = constant.Online - onlineStatus.UserID = userID - newjsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.WrapMsg(err, "json.Marshal failed") - } - _, err = u.rdb.HSet(ctx, key, userID, string(newjsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - - return nil -} - -type Comparable interface { - ~int | ~string | ~float64 | ~int32 -} - -func RemoveRepeatedElementsInList[T Comparable](slc []T) []T { - var result []T - tempMap := map[T]struct{}{} - for _, e := range slc { - if _, found := tempMap[e]; !found { - tempMap[e] = struct{}{} - result = append(result, e) - } - } - - return result -} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go deleted file mode 100644 index 091b318c89..0000000000 --- a/pkg/common/storage/cache/seq.go +++ /dev/null @@ -1,30 +0,0 @@ -package cache - -import ( - "context" -) - -type SeqCache interface { - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - // seqs map: key userID value minSeq - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) - // seqs map: key conversationID value minSeq - SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - // has read seq - SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - // k: user, v: seq - SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error - // k: conversation, v :seq - UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) - GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) -} diff --git a/pkg/common/storage/cache/seq_conversation.go b/pkg/common/storage/cache/seq_conversation.go new file mode 100644 index 0000000000..2c893a5e86 --- /dev/null +++ b/pkg/common/storage/cache/seq_conversation.go @@ -0,0 +1,12 @@ +package cache + +import "context" + +type SeqConversationCache interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + SetMinSeqs(ctx context.Context, seqs map[string]int64) error +} diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go new file mode 100644 index 0000000000..cef414e16e --- /dev/null +++ b/pkg/common/storage/cache/seq_user.go @@ -0,0 +1,16 @@ +package cache + +import "context" + +type SeqUser interface { + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserReadSeqToDB(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error + SetUserReadSeqs(ctx context.Context, userID string, seqs map[string]int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) +} diff --git a/pkg/common/storage/cache/user.go b/pkg/common/storage/cache/user.go index 5101c0b6ce..69a11635ca 100644 --- a/pkg/common/storage/cache/user.go +++ b/pkg/common/storage/cache/user.go @@ -17,7 +17,6 @@ package cache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/user" ) type UserCache interface { @@ -28,6 +27,6 @@ type UserCache interface { DelUsersInfo(userIDs ...string) UserCache GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error) DelUsersGlobalRecvMsgOpt(userIDs ...string) UserCache - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error + //GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) + //SetUserStatus(ctx context.Context, userID string, status, platformID int32) error } diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index fbfe30836a..b725513d9e 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -55,7 +55,6 @@ func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, p // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - isCreate := true // flag is create or update tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID) if err != nil { return "", err @@ -66,9 +65,6 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI if err != nil || v != constant.NormalToken { deleteTokenKey = append(deleteTokenKey, k) } - if v == constant.NormalToken { - isCreate = false - } } if len(deleteTokenKey) != 0 { err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) @@ -84,16 +80,8 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", errs.WrapMsg(err, "token.SignedString") } - if isCreate { - // should create,should specify expiration time - if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err - } - } else { - // should update - if err = a.cache.SetTokenFlag(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err - } + if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { + return "", err } return tokenString, nil } diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index 18ef3f8bad..c804d1cc51 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -66,6 +66,9 @@ type ConversationDatabase interface { GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) // GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) // FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) + FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) + GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -106,6 +109,7 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, if _, ok := fieldMap["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) } + cache = cache.DelConversationVersionUserIDs(haveUserIDs...) } NotUserIDs := stringutil.DifferenceString(haveUserIDs, userIDs) log.ZDebug(ctx, "SetUsersConversationFieldTx", "NotUserIDs", NotUserIDs, "haveUserIDs", haveUserIDs, "userIDs", userIDs) @@ -137,7 +141,7 @@ func (c *conversationDatabase) UpdateUsersConversationField(ctx context.Context, return err } cache := c.cache.CloneConversationCache() - cache = cache.DelUsersConversation(conversationID, userIDs...) + cache = cache.DelUsersConversation(conversationID, userIDs...).DelConversationVersionUserIDs(userIDs...) if _, ok := args["recv_msg_opt"]; ok { cache = cache.DelConversationNotReceiveMessageUserIDs(conversationID) } @@ -155,13 +159,14 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat cache = cache.DelConversationNotReceiveMessageUserIDs(conversation.ConversationID) userIDs = append(userIDs, conversation.OwnerUserID) } - return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).ChainExecDel(ctx) + return cache.DelConversationIDs(userIDs...).DelUserConversationIDsHash(userIDs...).DelConversationVersionUserIDs(userIDs...).ChainExecDel(ctx) } func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Context, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() for _, conversation := range conversations { + cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID) for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} { ownerUserID := v[0] userID := v[1] @@ -207,6 +212,7 @@ func (c *conversationDatabase) GetUserAllConversation(ctx context.Context, owner func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUserID string, conversations []*relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() + cache = cache.DelConversationVersionUserIDs(ownerUserID) groupIDs := datautil.Distinct(datautil.Filter(conversations, func(e *relationtb.Conversation) (string, bool) { return e.GroupID, e.GroupID != "" })) @@ -322,3 +328,28 @@ func (c *conversationDatabase) GetConversationIDsNeedDestruct(ctx context.Contex func (c *conversationDatabase) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { return c.cache.GetConversationNotReceiveMessageUserIDs(ctx, conversationID) } + +func (c *conversationDatabase) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*relationtb.VersionLog, error) { + return c.conversationDB.FindConversationUserVersion(ctx, userID, version, limit) +} + +func (c *conversationDatabase) FindMaxConversationUserVersionCache(ctx context.Context, userID string) (*relationtb.VersionLog, error) { + return c.cache.FindMaxConversationUserVersion(ctx, userID) +} + +func (c *conversationDatabase) GetOwnerConversation(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relationtb.Conversation, error) { + conversationIDs, err := c.cache.GetUserConversationIDs(ctx, ownerUserID) + if err != nil { + return 0, nil, err + } + findConversationIDs := datautil.Paginate(conversationIDs, int(pagination.GetPageNumber()), int(pagination.GetShowNumber())) + conversations := make([]*relationtb.Conversation, 0, len(findConversationIDs)) + for _, conversationID := range findConversationIDs { + conversation, err := c.cache.GetConversation(ctx, ownerUserID, conversationID) + if err != nil { + return 0, nil, err + } + conversations = append(conversations, conversation) + } + return int64(len(conversationIDs)), conversations, nil +} diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 1c3d9f139a..88a5fc863d 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -77,6 +77,16 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + + FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } type friendDatabase struct { @@ -142,42 +152,56 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return f.tx.Transaction(ctx, func(ctx context.Context) error { cache := f.cache.CloneFriendCache() // user find friends - fs1, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) + myFriends, err := f.friend.FindFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - opUserID := mcontext.GetOperationID(ctx) - for _, v := range friendUserIDs { - fs1 = append(fs1, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: v, AddSource: addSource, OperatorUserID: opUserID}) - } - fs11 := datautil.DistinctAny(fs1, func(e *model.Friend) string { - return e.FriendUserID - }) - - err = f.friend.Create(ctx, fs11) + addOwners, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) if err != nil { return err } - fs2, err := f.friend.FindReversalFriends(ctx, ownerUserID, friendUserIDs) - if err != nil { - return err + opUserID := mcontext.GetOpUserID(ctx) + friends := make([]*model.Friend, 0, len(friendUserIDs)*2) + myFriendsSet := datautil.SliceSetAny(myFriends, func(friend *model.Friend) string { + return friend.FriendUserID + }) + addOwnersSet := datautil.SliceSetAny(addOwners, func(friend *model.Friend) string { + return friend.OwnerUserID + }) + newMyFriendIDs := make([]string, 0, len(friendUserIDs)) + newMyOwnerIDs := make([]string, 0, len(friendUserIDs)) + for _, userID := range friendUserIDs { + if ownerUserID == userID { + continue + } + if _, ok := myFriendsSet[userID]; !ok { + myFriendsSet[userID] = struct{}{} + newMyFriendIDs = append(newMyFriendIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: ownerUserID, FriendUserID: userID, AddSource: addSource, OperatorUserID: opUserID}) + } + if _, ok := addOwnersSet[userID]; !ok { + addOwnersSet[userID] = struct{}{} + newMyOwnerIDs = append(newMyOwnerIDs, userID) + friends = append(friends, &model.Friend{OwnerUserID: userID, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) + } } - var newFriendIDs []string - for _, v := range friendUserIDs { - fs2 = append(fs2, &model.Friend{OwnerUserID: v, FriendUserID: ownerUserID, AddSource: addSource, OperatorUserID: opUserID}) - newFriendIDs = append(newFriendIDs, v) + if len(friends) == 0 { + return nil } - fs22 := datautil.DistinctAny(fs2, func(e *model.Friend) string { - return e.OwnerUserID - }) - err = f.friend.Create(ctx, fs22) + err = f.friend.Create(ctx, friends) if err != nil { return err } - newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...) + cache = cache.DelFriendIDs(ownerUserID).DelMaxFriendVersion(ownerUserID) + if len(newMyFriendIDs) > 0 { + cache = cache.DelFriendIDs(newMyFriendIDs...) + cache = cache.DelFriends(ownerUserID, newMyFriendIDs).DelMaxFriendVersion(newMyFriendIDs...) + } + if len(newMyOwnerIDs) > 0 { + cache = cache.DelFriendIDs(newMyOwnerIDs...) + cache = cache.DelOwner(ownerUserID, newMyOwnerIDs).DelMaxFriendVersion(newMyOwnerIDs...) + } return cache.ChainExecDel(ctx) - }) } @@ -278,7 +302,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelMaxFriendVersion(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -287,7 +311,8 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).ChainExecDel(ctx) + userIds := append(friendUserIDs, ownerUserID) + return f.cache.DelFriendIDs(userIds...).DelMaxFriendVersion(userIds...).ChainExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -295,7 +320,7 @@ func (f *friendDatabase) UpdateRemark(ctx context.Context, ownerUserID, friendUs if err := f.friend.UpdateRemark(ctx, ownerUserID, friendUserID, remark); err != nil { return err } - return f.cache.DelFriend(ownerUserID, friendUserID).ChainExecDel(ctx) + return f.cache.DelFriend(ownerUserID, friendUserID).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } // PageOwnerFriends retrieves the list of friends for the ownerUserID. It does not return an error if the result is empty. @@ -324,9 +349,6 @@ func (f *friendDatabase) FindFriendsWithError(ctx context.Context, ownerUserID s if err != nil { return } - if len(friends) != len(friendUserIDs) { - err = errs.ErrRecordNotFound.Wrap() - } return } @@ -341,8 +363,37 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if len(val) == 0 { return nil } - if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return err + } + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) + }) +} + +//func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +//} + +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) +} + +func (f *friendDatabase) FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return f.cache.FindMaxFriendVersion(ctx, ownerUserID) +} + +func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + return f.friend.FindFriendUserID(ctx, friendUserID) +} + +//func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +//} + +func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + if err := f.friend.IncrVersion(ctx, ownerUserID, friendUserIDs, state); err != nil { return err } - return f.cache.DelFriends(ownerUserID, friendUserIDs).ChainExecDel(ctx) + return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f2a1358357..072429ed09 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -16,17 +16,19 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" ) @@ -106,6 +108,22 @@ type GroupDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) // DeleteGroupMemberHash deletes the hash entries for group members in specified groups. DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error + + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error + + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) + BatchFindMaxGroupMemberVersionCache(ctx context.Context, groupIDs []string) (map[string]*model.VersionLog, error) + FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) + + SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) + + FindJoinGroupID(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -134,6 +152,10 @@ type groupDatabase struct { cache cache.GroupCache } +func (g *groupDatabase) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) { + return g.cache.GetJoinedGroupIDs(ctx, userID) +} + func (g *groupDatabase) FindGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { return g.cache.GetGroupMembersInfo(ctx, groupID, userIDs) } @@ -174,7 +196,8 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMembersHash(group.GroupID). DelGroupsMemberNum(group.GroupID). DelGroupMemberIDs(group.GroupID). - DelGroupAllRoleLevel(group.GroupID) + DelGroupAllRoleLevel(group.GroupID). + DelMaxGroupMemberVersion(group.GroupID) } } if len(groupMembers) > 0 { @@ -187,7 +210,9 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMemberIDs(groupMember.GroupID). DelJoinedGroupID(groupMember.UserID). DelGroupMembersInfo(groupMember.GroupID, groupMember.UserID). - DelGroupAllRoleLevel(groupMember.GroupID) + DelGroupAllRoleLevel(groupMember.GroupID). + DelMaxJoinGroupVersion(groupMember.UserID). + DelMaxGroupMemberVersion(groupMember.GroupID) } } return c.ChainExecDel(ctx) @@ -219,10 +244,15 @@ func (g *groupDatabase) SearchGroup(ctx context.Context, keyword string, paginat } func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data map[string]any) error { - if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { - return err - } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { + return err + } + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { + return err + } + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) + }) } func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error { @@ -244,7 +274,19 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupsMemberNum(groupID). DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). - DelGroupMembersInfo(groupID, userIDs...) + DelGroupMembersInfo(groupID, userIDs...). + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...) + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + } else { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { + return err + } + c = c.DelMaxGroupMemberVersion(groupID) } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) @@ -316,7 +358,9 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, DelGroupMemberIDs(groupID). DelGroupsMemberNum(groupID). DelJoinedGroupID(member.UserID). - DelGroupRoleLevel(groupID, []int32{member.RoleLevel}) + DelGroupRoleLevel(groupID, []int32{member.RoleLevel}). + DelMaxJoinGroupVersion(userID). + DelMaxGroupMemberVersion(groupID) if err := c.ChainExecDel(ctx); err != nil { return err } @@ -326,17 +370,21 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, } func (g *groupDatabase) DeleteGroupMember(ctx context.Context, groupID string, userIDs []string) error { - if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { - return err - } - c := g.cache.CloneGroupCache() - return c.DelGroupMembersHash(groupID). - DelGroupMemberIDs(groupID). - DelGroupsMemberNum(groupID). - DelJoinedGroupID(userIDs...). - DelGroupMembersInfo(groupID, userIDs...). - DelGroupAllRoleLevel(groupID). - ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { + return err + } + c := g.cache.CloneGroupCache() + return c.DelGroupMembersHash(groupID). + DelGroupMemberIDs(groupID). + DelGroupsMemberNum(groupID). + DelJoinedGroupID(userIDs...). + DelGroupMembersInfo(groupID, userIDs...). + DelGroupAllRoleLevel(groupID). + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...). + ChainExecDel(ctx) + }) } func (g *groupDatabase) MapGroupMemberUserID(ctx context.Context, groupIDs []string) (map[string]*common.GroupSimpleUserID, error) { @@ -357,29 +405,35 @@ func (g *groupDatabase) MapGroupMemberNum(ctx context.Context, groupIDs []string func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, oldOwnerUserID, newOwnerUserID string, roleLevel int32) error { return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { - if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, oldOwnerUserID, roleLevel); err != nil { - return err - } - if err := g.groupMemberDB.UpdateRoleLevel(ctx, groupID, newOwnerUserID, constant.GroupOwner); err != nil { + if err := g.groupMemberDB.UpdateUserRoleLevels(ctx, groupID, oldOwnerUserID, roleLevel, newOwnerUserID, constant.GroupOwner); err != nil { return err } c := g.cache.CloneGroupCache() return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). - DelGroupMembersHash(groupID).ChainExecDel(ctx) + DelGroupMembersHash(groupID). + DelMaxGroupMemberVersion(groupID). + DelGroupMemberIDs(groupID). + ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { - if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { - return err - } - c := g.cache.CloneGroupCache() - c = c.DelGroupMembersInfo(groupID, userID) - if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) + if len(data) == 0 { + return nil } - return c.ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { + return err + } + c := g.cache.CloneGroupCache() + c = c.DelGroupMembersInfo(groupID, userID) + if g.groupMemberDB.IsUpdateRoleLevel(data) { + c = c.DelGroupAllRoleLevel(groupID).DelGroupMemberIDs(groupID) + } + c = c.DelMaxGroupMemberVersion(groupID) + return c.ChainExecDel(ctx) + }) } func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.BatchUpdateGroupMember) error { @@ -390,9 +444,9 @@ func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.B return err } if g.groupMemberDB.IsUpdateRoleLevel(item.Map) { - c = c.DelGroupAllRoleLevel(item.GroupID) + c = c.DelGroupAllRoleLevel(item.GroupID).DelGroupMemberIDs(item.GroupID) } - c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelGroupMembersHash(item.GroupID) + c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelMaxGroupMemberVersion(item.GroupID).DelGroupMembersHash(item.GroupID) } return c.ChainExecDel(ctx) }) @@ -443,3 +497,71 @@ func (g *groupDatabase) DeleteGroupMemberHash(ctx context.Context, groupIDs []st } return c.ChainExecDel(ctx) } + +func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit) +} + +func (g *groupDatabase) BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { + if len(groupIDs) == 0 { + return nil, errs.Wrap(errs.New("groupIDs is nil.")) + } + + // convert []uint64 to []uint + var uintVersions []uint + for _, version := range versions { + uintVersions = append(uintVersions, uint(version)) + } + + versionLogs, err := g.groupMemberDB.BatchFindMemberIncrVersion(ctx, groupIDs, uintVersions, limits) + if err != nil { + return nil, errs.Wrap(err) + } + + groupMemberIncrVersionsMap := datautil.SliceToMap(versionLogs, func(e *model.VersionLog) string { + return e.DID + }) + + return groupMemberIncrVersionsMap, nil +} + +func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) +} + +func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { + return g.cache.FindMaxGroupMemberVersion(ctx, groupID) +} + +func (g *groupDatabase) BatchFindMaxGroupMemberVersionCache(ctx context.Context, groupIDs []string) (map[string]*model.VersionLog, error) { + if len(groupIDs) == 0 { + return nil, errs.Wrap(errs.New("groupIDs is nil in Cache.")) + } + versionLogs, err := g.cache.BatchFindMaxGroupMemberVersion(ctx, groupIDs) + if err != nil { + return nil, errs.Wrap(err) + } + maxGroupMemberVersionsMap := datautil.SliceToMap(versionLogs, func(e *model.VersionLog) string { + return e.DID + }) + return maxGroupMemberVersionsMap, nil +} + +func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { + return g.cache.FindMaxJoinGroupVersion(ctx, userID) +} + +func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + groupIDs, err := g.cache.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return 0, nil, err + } + return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) +} + +func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, userIDs, state); err != nil { + return err + } + return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8eb9e8e6fd..fdd06d3ff3 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,14 +18,14 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "strings" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" @@ -46,16 +46,10 @@ const ( // CommonMsgDatabase defines the interface for message database operations. type CommonMsgDatabase interface { - // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. - BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error // RevokeMsg revokes a message in a conversation. RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error // MarkSingleChatMsgsAsRead marks messages as read for a single chat by sequence numbers. MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error - // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. - DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error - // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. - BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) // GetMsgBySeqsRange retrieves messages from MongoDB by a range of sequence numbers. GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // GetMsgBySeqs retrieves messages for large groups from MongoDB by sequence numbers. @@ -69,46 +63,39 @@ type CommonMsgDatabase interface { DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, seqs []int64) error - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) + //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) + //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) + SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) FindOneByDocIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) // to mq MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error - MsgToPushMQ(ctx context.Context, key, conversarionID string, msg2mq *sdkws.MsgData) (int32, int64, error) - MsgToMongoMQ(ctx context.Context, key, conversarionID string, msgs []*sdkws.MsgData, lastSeq int64) error RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // clear msg - GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) + GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + + GetDocIDs(ctx context.Context) ([]string, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -117,43 +104,22 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach if err != nil { return nil, err } - producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) - if err != nil { - return nil, err - } - producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) - if err != nil { - return nil, err - } return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, - seq: seq, + seqUser: seqUser, + seqConversation: seqConversation, producer: producerToRedis, - producerToMongo: producerToMongo, - producerToPush: producerToPush, }, nil } -//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) { -// msgDocModel, err := database.NewMsgMongo(database) -// if err != nil { -// return nil, err -// } -// //todo MsgCacheTimeout -// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline) -// seq := cache.NewSeqCache(rdb) -// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig) -//} - type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache - seq cache.SeqCache + seqConversation cache.SeqConversationCache + seqUser cache.SeqUser producer *kafka.Producer - producerToMongo *kafka.Producer - producerToPush *kafka.Producer } func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error { @@ -161,23 +127,6 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd return err } -func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { - partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) - if err != nil { - log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) - return 0, 0, err - } - return partition, offset, nil -} - -func (db *commonMsgDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { - if len(messages) > 0 { - _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) - return err - } - return nil -} - func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { if len(fields) == 0 { return nil @@ -279,52 +228,6 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI return nil } -func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { - if len(msgList) == 0 { - return errs.ErrArgs.WrapMsg("msgList is empty") - } - msgs := make([]any, len(msgList)) - for i, msg := range msgList { - if msg == nil { - continue - } - var offlinePushModel *model.OfflinePushModel - if msg.OfflinePushInfo != nil { - offlinePushModel = &model.OfflinePushModel{ - Title: msg.OfflinePushInfo.Title, - Desc: msg.OfflinePushInfo.Desc, - Ex: msg.OfflinePushInfo.Ex, - IOSPushSound: msg.OfflinePushInfo.IOSPushSound, - IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, - } - } - msgs[i] = &model.MsgDataModel{ - SendID: msg.SendID, - RecvID: msg.RecvID, - GroupID: msg.GroupID, - ClientMsgID: msg.ClientMsgID, - ServerMsgID: msg.ServerMsgID, - SenderPlatformID: msg.SenderPlatformID, - SenderNickname: msg.SenderNickname, - SenderFaceURL: msg.SenderFaceURL, - SessionType: msg.SessionType, - MsgFrom: msg.MsgFrom, - ContentType: msg.ContentType, - Content: string(msg.Content), - Seq: msg.Seq, - SendTime: msg.SendTime, - CreateTime: msg.CreateTime, - Status: msg.Status, - Options: msg.Options, - OfflinePush: offlinePushModel, - AtUserIDList: msg.AtUserIDList, - AttachedInfo: msg.AttachedInfo, - Ex: msg.Ex, - } - } - return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) -} - func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error { return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq) } @@ -344,56 +247,6 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return nil } -func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) -} - -func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { - log.ZError(ctx, "storage.seq.GetMaxSeq", err) - return 0, false, err - } - lenList := len(msgs) - if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { - return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() - } - if lenList < 1 { - return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() - } - if errs.Unwrap(err) == redis.Nil { - isNew = true - } - lastMaxSeq := currentMaxSeq - userSeqMap := make(map[string]int64) - for _, m := range msgs { - currentMaxSeq++ - m.Seq = currentMaxSeq - userSeqMap[m.SendID] = m.Seq - } - - failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) - if err != nil { - prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) - log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) - } else { - prommetrics.MsgInsertRedisSuccessCounter.Inc() - } - - err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq) - if err != nil { - log.ZError(ctx, "storage.seq.SetMaxSeq error", err, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) - if err != nil { - log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - return lastMaxSeq, isNew, errs.Wrap(err) -} - func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { // log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs) @@ -514,12 +367,12 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq > minSeq { @@ -530,8 +383,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end) return 0, 0, nil, nil } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq) @@ -571,11 +424,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin var successMsgs []*sdkws.MsgData log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil { - if err != redis.Nil { - - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } + if err != nil && !errors.Is(err, redis.Nil) { + log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) } successMsgs = append(successMsgs, cachedMsgs...) log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) @@ -595,16 +445,16 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq < minSeq { @@ -648,7 +498,7 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont if minSeq == 0 { return nil } - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) { @@ -693,12 +543,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) + if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -740,7 +590,7 @@ func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversatio } log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg)) if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() { - log.ZWarn(ctx, "msgs too large", nil, "lenth", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) + log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) } if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() { log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID) @@ -796,89 +646,40 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st return nil } -func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationID string, seqs []int64) error { - return nil -} - -func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) { - for _, conversationID := range conversationIDs { - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil { - if err == redis.Nil { - log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID) - } else { - log.ZError(ctx, "get max seq failed", err, "conversationID", conversationID) - } - continue - } - if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { - log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1) - } - } -} - -func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) -} - func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMaxSeqs(ctx, conversationIDs) + return db.seqConversation.GetMaxSeqs(ctx, conversationIDs) } func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMaxSeq(ctx, conversationID) + return db.seqConversation.GetMaxSeq(ctx, conversationID) } func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return db.seq.SetMinSeqs(ctx, seqs) -} - -func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMinSeqs(ctx, conversationIDs) -} - -func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMinSeq(ctx, conversationID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) { - return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs) + return db.seqConversation.SetMinSeqs(ctx, seqs) } func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs) + return db.seqUser.SetUserMinSeqs(ctx, userID, seqs) } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs) + return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq) + return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs) + return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seq.GetHasReadSeq(ctx, userID, conversationID) + return db.seqUser.GetUserReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { @@ -894,11 +695,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context if err != nil { return } - minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID) + minSeqCache, err = db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { return } - maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID) + maxSeqCache, err = db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { return } @@ -946,7 +747,7 @@ func (db *commonMsgDatabase) RangeGroupSendCount( return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) { +func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) { var totalMsgs []*sdkws.MsgData total, msgs, err := db.msgDocDatabase.SearchMessage(ctx, req) if err != nil { @@ -980,8 +781,25 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { - return db.msgDocDatabase.GetBeforeMsg(ctx, ts, limit) +func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { + var msgs []*model.MsgDocModel + for i := 0; i < len(docIDs); i += 1000 { + end := i + 1000 + if end > len(docIDs) { + end = len(docIDs) + } + + res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) + if err != nil { + return nil, err + } + msgs = append(msgs, res...) + + if len(msgs) >= limit { + return msgs[:limit], nil + } + } + return msgs, nil } func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { @@ -1004,39 +822,16 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d return index, err } if len(index) == notNull { + log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) } else { + log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) } } -//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) { -// var ( -// docNum int -// msgNum int -// start = time.Now() -// ) -// for { -// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return err -// } -// if len(msgs) == 0 { -// return nil -// } -// for _, msg := range msgs { -// num, err := db.deleteOneMsg(ctx, ts, msg) -// if err != nil { -// return err -// } -// docNum++ -// msgNum += num -// } -// } -//} - func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { - dbSeq, err := db.seq.GetMinSeq(ctx, conversationID) + dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { return nil @@ -1046,5 +841,9 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin if dbSeq >= seq { return nil } - return db.seq.SetMinSeq(ctx, conversationID, seq) + return db.seqConversation.SetMinSeq(ctx, conversationID, seq) +} + +func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { + return db.msgDocDatabase.GetDocIDs(ctx) } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go new file mode 100644 index 0000000000..5e540a2c33 --- /dev/null +++ b/pkg/common/storage/controller/msg_transfer.go @@ -0,0 +1,286 @@ +package controller + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + pbmsg "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "go.mongodb.org/mongo-driver/mongo" +) + +type MsgTransferDatabase interface { + // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. + BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error + // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. + DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error + + // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. + BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) + SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + + // to mq + MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) + MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error +} + +func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (MsgTransferDatabase, error) { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil, err + } + producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) + if err != nil { + return nil, err + } + producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) + if err != nil { + return nil, err + } + return &msgTransferDatabase{ + msgDocDatabase: msgDocModel, + msg: msg, + seqUser: seqUser, + seqConversation: seqConversation, + producerToMongo: producerToMongo, + producerToPush: producerToPush, + }, nil +} + +type msgTransferDatabase struct { + msgDocDatabase database.Msg + msgTable model.MsgDocModel + msg cache.MsgCache + seqConversation cache.SeqConversationCache + seqUser cache.SeqUser + producerToMongo *kafka.Producer + producerToPush *kafka.Producer +} + +func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { + if len(msgList) == 0 { + return errs.ErrArgs.WrapMsg("msgList is empty") + } + msgs := make([]any, len(msgList)) + for i, msg := range msgList { + if msg == nil { + continue + } + var offlinePushModel *model.OfflinePushModel + if msg.OfflinePushInfo != nil { + offlinePushModel = &model.OfflinePushModel{ + Title: msg.OfflinePushInfo.Title, + Desc: msg.OfflinePushInfo.Desc, + Ex: msg.OfflinePushInfo.Ex, + IOSPushSound: msg.OfflinePushInfo.IOSPushSound, + IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, + } + } + msgs[i] = &model.MsgDataModel{ + SendID: msg.SendID, + RecvID: msg.RecvID, + GroupID: msg.GroupID, + ClientMsgID: msg.ClientMsgID, + ServerMsgID: msg.ServerMsgID, + SenderPlatformID: msg.SenderPlatformID, + SenderNickname: msg.SenderNickname, + SenderFaceURL: msg.SenderFaceURL, + SessionType: msg.SessionType, + MsgFrom: msg.MsgFrom, + ContentType: msg.ContentType, + Content: string(msg.Content), + Seq: msg.Seq, + SendTime: msg.SendTime, + CreateTime: msg.CreateTime, + Status: msg.Status, + Options: msg.Options, + OfflinePush: offlinePushModel, + AtUserIDList: msg.AtUserIDList, + AttachedInfo: msg.AttachedInfo, + Ex: msg.Ex, + } + } + return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) +} + +func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { + if len(fields) == 0 { + return nil + } + num := db.msgTable.GetSingleGocMsgNum() + // num = 100 + for i, field := range fields { // Check the type of the field + var ok bool + switch key { + case updateKeyMsg: + var msg *model.MsgDataModel + msg, ok = field.(*model.MsgDataModel) + if msg != nil && msg.Seq != firstSeq+int64(i) { + return errs.ErrInternalServer.WrapMsg("seq is invalid") + } + case updateKeyRevoke: + _, ok = field.(*model.RevokeModel) + default: + return errs.ErrInternalServer.WrapMsg("key is invalid") + } + if !ok { + return errs.ErrInternalServer.WrapMsg("field type is invalid") + } + } + // Returns true if the document exists in the database, false if the document does not exist in the database + updateMsgModel := func(seq int64, i int) (bool, error) { + var ( + res *mongo.UpdateResult + err error + ) + docID := db.msgTable.GetDocID(conversationID, seq) + index := db.msgTable.GetMsgIndex(seq) + field := fields[i] + switch key { + case updateKeyMsg: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "msg", field) + case updateKeyRevoke: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "revoke", field) + } + if err != nil { + return false, err + } + return res.MatchedCount > 0, nil + } + tryUpdate := true + for i := 0; i < len(fields); i++ { + seq := firstSeq + int64(i) // Current sequence number + if tryUpdate { + matched, err := updateMsgModel(seq, i) + if err != nil { + return err + } + if matched { + continue // The current data has been updated, skip the current data + } + } + doc := model.MsgDocModel{ + DocID: db.msgTable.GetDocID(conversationID, seq), + Msg: make([]*model.MsgInfoModel, num), + } + var insert int // Inserted data number + for j := i; j < len(fields); j++ { + seq = firstSeq + int64(j) + if db.msgTable.GetDocID(conversationID, seq) != doc.DocID { + break + } + insert++ + switch key { + case updateKeyMsg: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Msg: fields[j].(*model.MsgDataModel), + } + case updateKeyRevoke: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Revoke: fields[j].(*model.RevokeModel), + } + } + } + for i, msgInfo := range doc.Msg { + if msgInfo == nil { + msgInfo = &model.MsgInfoModel{} + doc.Msg[i] = msgInfo + } + if msgInfo.DelList == nil { + doc.Msg[i].DelList = []string{} + } + } + if err := db.msgDocDatabase.Create(ctx, &doc); err != nil { + if mongo.IsDuplicateKeyError(err) { + i-- // already inserted + tryUpdate = true // next block use update mode + continue + } + return err + } + tryUpdate = false // The current block is inserted successfully, and the next block is inserted preferentially + i += insert - 1 // Skip the inserted data + } + return nil +} + +func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { + return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) +} + +func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { + lenList := len(msgs) + if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { + return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() + } + if lenList < 1 { + return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() + } + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err + } + isNew = currentMaxSeq == 0 + lastMaxSeq := currentMaxSeq + userSeqMap := make(map[string]int64) + for _, m := range msgs { + currentMaxSeq++ + m.Seq = currentMaxSeq + userSeqMap[m.SendID] = m.Seq + } + + failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) + if err != nil { + prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) + log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) + } else { + prommetrics.MsgInsertRedisSuccessCounter.Inc() + } + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) + if err != nil { + log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) + prommetrics.SeqSetFailedCounter.Inc() + } + return lastMaxSeq, isNew, errs.Wrap(err) +} + +func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil +} + +func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { + return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +} + +func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { + partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) + if err != nil { + log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) + return 0, 0, err + } + return partition, offset, nil +} + +func (db *msgTransferDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { + if len(messages) > 0 { + _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) + if err != nil { + log.ZError(ctx, "MsgToMongoMQ", err, "key", key, "conversationID", conversationID, "lastSeq", lastSeq) + return err + } + } + return nil +} diff --git a/pkg/common/storage/controller/push.go b/pkg/common/storage/controller/push.go index 199a0ba678..91ef126fe5 100644 --- a/pkg/common/storage/controller/push.go +++ b/pkg/common/storage/controller/push.go @@ -17,21 +17,45 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" ) type PushDatabase interface { DelFcmToken(ctx context.Context, userID string, platformID int) error + MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error } type pushDataBase struct { - cache cache.ThirdCache + cache cache.ThirdCache + producerToOfflinePush *kafka.Producer } -func NewPushDatabase(cache cache.ThirdCache) PushDatabase { - return &pushDataBase{cache: cache} +func NewPushDatabase(cache cache.ThirdCache, kafkaConf *config.Kafka) PushDatabase { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil + } + producerToOfflinePush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToOfflinePushTopic) + if err != nil { + return nil + } + return &pushDataBase{ + cache: cache, + producerToOfflinePush: producerToOfflinePush, + } } func (p *pushDataBase) DelFcmToken(ctx context.Context, userID string, platformID int) error { return p.cache.DelFcmToken(ctx, userID, platformID) } + +func (p *pushDataBase) MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error { + _, _, err := p.producerToOfflinePush.SendMessage(ctx, key, &push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs}) + log.ZInfo(ctx, "message is push to offlinePush topic", "key", key, "userIDs", userIDs, "msg", msg2mq.String()) + return err +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b0ad612038..9b56661a59 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -16,13 +16,15 @@ package controller import ( "context" - redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path/filepath" "time" + redisCache "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -38,20 +40,27 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + DeleteObject(ctx context.Context, name string) error + DeleteSpecifiedData(ctx context.Context, engine string, name string) error + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + DelS3Key(ctx context.Context, engine string, keys ...string) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { return &s3Database{ - s3: cont.New(redis2.NewS3Cache(rdb, s3), s3), - cache: redis2.NewObjectCacheRedis(rdb, obj), - db: obj, + s3: cont.New(redisCache.NewS3Cache(rdb, s3), s3), + cache: redisCache.NewObjectCacheRedis(rdb, obj), + s3cache: redisCache.NewS3Cache(rdb, s3), + db: obj, } } type s3Database struct { - s3 *cont.Controller - cache cache.ObjectCache - db database.ObjectInfo + s3 *cont.Controller + cache cache.ObjectCache + s3cache cont.S3Cache + db database.ObjectInfo } func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) { @@ -111,3 +120,22 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + + return s.db.FindByExpires(ctx, duration, pagination) +} + +func (s *s3Database) DeleteObject(ctx context.Context, name string) error { + return s.s3.DeleteObject(ctx, name) +} +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { + return s.db.Delete(ctx, engine, name) +} + +func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return s.db.FindNotDelByS3(ctx, key, duration) +} + +func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { + return s.s3cache.DelS3Key(ctx, engine, keys...) +} diff --git a/pkg/common/storage/controller/third.go b/pkg/common/storage/controller/third.go index 344501466b..a9c2ae403c 100644 --- a/pkg/common/storage/controller/third.go +++ b/pkg/common/storage/controller/third.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/db/pagination" diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 09dc2db22b..533eac78f1 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -60,18 +60,8 @@ type UserDatabase interface { CountTotal(ctx context.Context, before *time.Time) (int64, error) // CountRangeEverydayTotal Get the user increment in the range CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) - // SubscribeUsersStatus Subscribe a user's presence status - SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // UnsubscribeUsersStatus unsubscribe a user's presence status - UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error - // GetAllSubscribeList Get a list of all subscriptions - GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) - // GetSubscribedList Get all subscribed lists - GetSubscribedList(ctx context.Context, userID string) ([]string, error) - // GetUserStatus Get the online status of the user - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - // SetUserStatus Set the user status and store the user status in redis - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -82,14 +72,13 @@ type UserDatabase interface { } type userDatabase struct { - tx tx.Tx - userDB database.User - cache cache.UserCache - mongoDB database.SubscribeUser + tx tx.Tx + userDB database.User + cache cache.UserCache } -func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx, mongoDB database.SubscribeUser) UserDatabase { - return &userDatabase{userDB: userDB, cache: cache, tx: tx, mongoDB: mongoDB} +func NewUserDatabase(userDB database.User, cache cache.UserCache, tx tx.Tx) UserDatabase { + return &userDatabase{userDB: userDB, cache: cache, tx: tx} } func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error { @@ -121,6 +110,7 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error // FindWithError Get the information of the specified user and return an error if the userID is not found. func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (users []*model.User, err error) { + userIDs = datautil.Distinct(userIDs) users, err = u.cache.GetUsersInfo(ctx, userIDs) if err != nil { return @@ -197,7 +187,7 @@ func (u *userDatabase) GetAllUserID(ctx context.Context, pagination pagination.P } func (u *userDatabase) GetUserByID(ctx context.Context, userID string) (user *model.User, err error) { - return u.userDB.Take(ctx, userID) + return u.cache.GetUserInfo(ctx, userID) } // CountTotal Get the total number of users. @@ -210,45 +200,8 @@ func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.T return u.userDB.CountRangeEverydayTotal(ctx, start, end) } -// SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. -func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) - return err -} - -// UnsubscribeUsersStatus unsubscribe a user's presence status. -func (u *userDatabase) UnsubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { - err := u.mongoDB.UnsubscriptionList(ctx, userID, userIDs) - return err -} - -// GetAllSubscribeList Get a list of all subscriptions. -func (u *userDatabase) GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetAllSubscribeList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - -// GetSubscribedList Get all subscribed lists. -func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([]string, error) { - list, err := u.mongoDB.GetSubscribedList(ctx, userID) - if err != nil { - return nil, err - } - return list, nil -} - -// GetUserStatus get user status. -func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - onlineStatusList, err := u.cache.GetUserStatus(ctx, userIDs) - return onlineStatusList, err -} - -// SetUserStatus Set the user status and save it in redis. -func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - return u.cache.SetUserStatus(ctx, userID, status, platformID) +func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + return u.userDB.SortQuery(ctx, userIDName, asc) } func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 46aa02d98e..85f3dd668a 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -22,7 +22,6 @@ import ( type Conversation interface { Create(ctx context.Context, conversations []*model.Conversation) (err error) - Delete(ctx context.Context, groupIDs []string) (err error) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) Update(ctx context.Context, conversation *model.Conversation) (err error) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) @@ -39,4 +38,5 @@ type Conversation interface { GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*model.Conversation, error) GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) + FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 33d9c17bc5..b596411fce 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -46,4 +47,14 @@ type Friend interface { FindFriendUserIDs(ctx context.Context, ownerUserID string) (friendUserIDs []string, err error) // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) + + IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } diff --git a/pkg/common/storage/database/group.go b/pkg/common/storage/database/group.go index 712db09d2d..7ef22f6c9a 100644 --- a/pkg/common/storage/database/group.go +++ b/pkg/common/storage/database/group.go @@ -32,4 +32,8 @@ type Group interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get Group total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) + + SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index f57f2c3173..0ddf0654c0 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -25,8 +26,11 @@ type GroupMember interface { Delete(ctx context.Context, groupID string, userIDs []string) (err error) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error + UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) + Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) + FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) TakeOwner(ctx context.Context, groupID string) (groupMember *model.GroupMember, err error) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) FindRoleLevelUserIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) @@ -34,4 +38,9 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint, limits []int) ([]*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/black.go b/pkg/common/storage/database/mgo/black.go index cf74cfab14..4a7a35e6f1 100644 --- a/pkg/common/storage/database/mgo/black.go +++ b/pkg/common/storage/database/mgo/black.go @@ -27,7 +27,7 @@ import ( ) func NewBlackMongo(db *mongo.Database) (database.Black, error) { - coll := db.Collection("black") + coll := db.Collection(database.BlackName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 9c35f841b6..3d505f1d34 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" @@ -29,7 +30,7 @@ import ( ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { - coll := db.Collection("conversation") + coll := db.Collection(database.ConversationName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, @@ -40,40 +41,71 @@ func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { if err != nil { return nil, errs.Wrap(err) } - return &ConversationMgo{coll: coll}, nil + version, err := NewVersionLog(db.Collection(database.ConversationVersionName)) + if err != nil { + return nil, err + } + return &ConversationMgo{version: version, coll: coll}, nil } type ConversationMgo struct { - coll *mongo.Collection + version database.VersionLog + coll *mongo.Collection } func (c *ConversationMgo) Create(ctx context.Context, conversations []*model.Conversation) (err error) { - return mongoutil.InsertMany(ctx, c.coll, conversations) -} - -func (c *ConversationMgo) Delete(ctx context.Context, groupIDs []string) (err error) { - return mongoutil.DeleteMany(ctx, c.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, c.coll, conversations) + }, func() error { + userConversation := make(map[string][]string) + for _, conversation := range conversations { + userConversation[conversation.OwnerUserID] = append(userConversation[conversation.OwnerUserID], conversation.ConversationID) + } + for userID, conversationIDs := range userConversation { + if err := c.version.IncrVersion(ctx, userID, conversationIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } -func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) { - if len(args) == 0 { +func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (int64, error) { + if len(args) == 0 || len(userIDs) == 0 { return 0, nil } filter := bson.M{ "conversation_id": conversationID, + "owner_user_id": bson.M{"$in": userIDs}, } - if len(userIDs) > 0 { - filter["owner_user_id"] = bson.M{"$in": userIDs} - } - res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + var rows int64 + err := mongoutil.IncrVersion(func() error { + res, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + if err != nil { + return err + } + rows = res.ModifiedCount + return nil + }, func() error { + for _, userID := range userIDs { + if err := c.version.IncrVersion(ctx, userID, []string{conversationID}, model.VersionStateUpdate); err != nil { + return err + } + } + return nil + }) if err != nil { return 0, err } - return res.ModifiedCount, nil + return rows, nil } func (c *ConversationMgo) Update(ctx context.Context, conversation *model.Conversation) (err error) { - return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) + }, func() error { + return c.version.IncrVersion(ctx, conversation.OwnerUserID, []string{conversation.ConversationID}, model.VersionStateUpdate) + }) } func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) { @@ -177,3 +209,7 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}), ) } + +func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return c.version.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index ffa006d013..76c82bac24 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -18,6 +18,8 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" @@ -28,12 +30,13 @@ import ( // FriendMgo implements Friend using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection + coll *mongo.Collection + owner database.VersionLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. func NewFriendMongo(db *mongo.Database) (database.Friend, error) { - coll := db.Collection("friend") + coll := db.Collection(database.FriendName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, @@ -44,12 +47,41 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - return &FriendMgo{coll: coll}, nil + owner, err := NewVersionLog(db.Collection(database.FriendVersionName)) + if err != nil { + return nil, err + } + return &FriendMgo{coll: coll, owner: owner}, nil +} + +func (f *FriendMgo) friendSort() any { + return bson.D{{"is_pinned", -1}, {"_id", 1}} } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { - return mongoutil.InsertMany(ctx, f.coll, friends) + for i, friend := range friends { + if friend.ID.IsZero() { + friends[i].ID = primitive.NewObjectID() + } + if friend.CreateTime.IsZero() { + friends[i].CreateTime = time.Now() + } + } + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, f.coll, friends) + }, func() error { + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) + } + for ownerUserID, friendUserIDs := range mp { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } // Delete removes specified friends of the owner user. @@ -58,11 +90,15 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.DeleteOne(ctx, f.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteOne(ctx, f.coll, filter) + }, func() error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete) + }) } // UpdateByMap updates specific fields of a friend document using a map. -func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]interface{}) error { +func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendUserID string, args map[string]any) error { if len(args) == 0 { return nil } @@ -70,30 +106,61 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + }, func() error { + var friendUserIDs []string + if f.IsUpdateIsPinned(args) { + friendUserIDs = []string{model.VersionSortChangeID, friendUserID} + } else { + friendUserIDs = []string{friendUserID} + } + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) + }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) } +func (f *FriendMgo) fillTime(friends ...*model.Friend) { + for i, friend := range friends { + if friend.CreateTime.IsZero() { + friends[i].CreateTime = friend.ID.Timestamp() + } + } +} + +func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) { + friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friend) + return friend, nil +} + +func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) { + friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friends...) + return friends, nil +} + +func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) { + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...) +} + // Take retrieves a single friend document. Returns an error if not found. func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) { filter := bson.M{ "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + return f.findOne(ctx, filter) } // FindUserState finds the friendship status between two users. @@ -104,7 +171,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string) {"owner_user_id": userID2, "friend_user_id": userID1}, }, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error. @@ -113,7 +180,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindReversalFriends finds users who have added the specified user as a friend. @@ -122,30 +189,38 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string "owner_user_id": bson.M{"$in": ownerUserIDs}, "friend_user_id": friendUserID, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) +} + +func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { + filter := bson.M{"owner_user_id": ownerUserID} + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit)) + return mongoutil.Find[string](ctx, f.coll, filter, opt) } // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"friend_user_id": friendUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) } // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { // Ensure there are IDs to update - if len(friendUserIDs) == 0 { + if len(friendUserIDs) == 0 || len(val) == 0 { return nil // Or return an error if you expect there to always be IDs } @@ -158,7 +233,38 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - // Perform the update operation for all matching documents - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, update) - return err + return mongoutil.IncrVersion(func() error { + return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) + }, func() error { + var userIDs []string + if f.IsUpdateIsPinned(val) { + userIDs = append([]string{model.VersionSortChangeID}, friendUserIDs...) + } else { + userIDs = friendUserIDs + } + return f.owner.IncrVersion(ctx, ownerUserID, userIDs, model.VersionStateUpdate) + }) +} + +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { + return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) +} + +func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + filter := bson.M{ + "friend_user_id": friendUserID, + } + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) +} + +func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) +} + +func (f *FriendMgo) IsUpdateIsPinned(data map[string]any) bool { + if data == nil { + return false + } + _, ok := data["is_pinned"] + return ok } diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 0d60b213dd..4eed2f4a26 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -27,7 +27,7 @@ import ( ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { - coll := db.Collection("friend_request") + coll := db.Collection(database.FriendRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "from_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 48d24560bc..3be7883af9 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -30,7 +30,7 @@ import ( ) func NewGroupMongo(db *mongo.Database) (database.Group, error) { - coll := db.Collection("group") + coll := db.Collection(database.GroupName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -47,6 +47,10 @@ type GroupMgo struct { coll *mongo.Collection } +func (g *GroupMgo) sortGroup() any { + return bson.D{{"group_name", 1}, {"create_time", 1}} +} + func (g *GroupMgo) Create(ctx context.Context, groups []*model.Group) (err error) { return mongoutil.InsertMany(ctx, g.coll, groups) } @@ -126,3 +130,32 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (g *GroupMgo) FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) { + if len(groupIDs) < 2 { + return groupIDs, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + opt := options.Find().SetSort(g.sortGroup()).SetProjection(bson.M{"_id": 0, "group_id": 1}) + return mongoutil.Find[string](ctx, g.coll, filter, opt) +} + +func (g *GroupMgo) SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + if len(groupIDs) == 0 { + return 0, nil, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + if keyword != "" { + filter["group_name"] = bson.M{"$regex": keyword} + } + // Define the sorting options + opts := options.Find().SetSort(g.sortGroup()) + // Perform the search with pagination and sorting + return mongoutil.FindPage[*model.Group](ctx, g.coll, filter, pagination, opts) +} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ccca386e57..2fdf2003b5 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -16,8 +16,10 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/log" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" @@ -29,7 +31,7 @@ import ( ) func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { - coll := db.Collection("group_member") + coll := db.Collection(database.GroupMemberName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -40,15 +42,53 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - return &GroupMemberMgo{coll: coll}, nil + member, err := NewVersionLog(db.Collection(database.GroupMemberVersionName)) + if err != nil { + return nil, err + } + join, err := NewVersionLog(db.Collection(database.GroupJoinVersionName)) + if err != nil { + return nil, err + } + return &GroupMemberMgo{coll: coll, member: member, join: join}, nil } type GroupMemberMgo struct { - coll *mongo.Collection + coll *mongo.Collection + member database.VersionLog + join database.VersionLog +} + +func (g *GroupMemberMgo) memberSort() any { + return bson.D{{"role_level", -1}, {"create_time", 1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { - return mongoutil.InsertMany(ctx, g.coll, groupMembers) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, g.coll, groupMembers) + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.GroupID] = append(gms[member.GroupID], member.UserID) + } + for groupID, userIDs := range gms { + if err := g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.UserID] = append(gms[member.UserID], member.GroupID) + } + for userID, groupIDs := range gms { + if err := g.join.IncrVersion(ctx, userID, groupIDs, model.VersionStateInsert); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) { @@ -56,24 +96,83 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s if len(userIDs) > 0 { filter["user_id"] = bson.M{"$in": userIDs} } - return mongoutil.DeleteMany(ctx, g.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteMany(ctx, g.coll, filter) + }, func() error { + if len(userIDs) == 0 { + return g.member.Delete(ctx, groupID) + } else { + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + } + }, func() error { + for _, userID := range userIDs { + if err := g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error { - return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, + bson.M{"$set": bson.M{"role_level": roleLevel}}, true) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, userID}, model.VersionStateUpdate) + }) +} +func (g *GroupMemberMgo) UpdateUserRoleLevels(ctx context.Context, groupID string, firstUserID string, firstUserRoleLevel int32, secondUserID string, secondUserRoleLevel int32) error { + return mongoutil.IncrVersion(func() error { + if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": firstUserID}, + bson.M{"$set": bson.M{"role_level": firstUserRoleLevel}}, true); err != nil { + return err + } + if err := mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": secondUserID}, + bson.M{"$set": bson.M{"role_level": secondUserRoleLevel}}, true); err != nil { + return err + } + return nil + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{model.VersionSortChangeID, firstUserID, secondUserID}, model.VersionStateUpdate) + }) } func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) { - return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) + if len(data) == 0 { + return nil + } + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) + }, func() error { + var userIDs []string + if g.IsUpdateRoleLevel(data) { + userIDs = []string{model.VersionSortChangeID, userID} + } else { + userIDs = []string{userID} + } + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateUpdate) + }) } -func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*model.GroupMember, err error) { - // TODO implement me - panic("implement me") +func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } -func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) +func (g *GroupMemberMgo) Find(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"group_id": groupID} + if len(userIDs) > 0 { + filter["user_id"] = bson.M{"$in": userIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) +} + +func (g *GroupMemberMgo) FindInGroup(ctx context.Context, userID string, groupIDs []string) ([]*model.GroupMember, error) { + filter := bson.M{"user_id": userID} + if len(groupIDs) > 0 { + filter["group_id"] = bson.M{"$in": groupIDs} + } + return mongoutil.Find[*model.GroupMember](ctx, g.coll, filter) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -88,13 +187,13 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) } -func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { +func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (int64, []*model.GroupMember, error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.memberSort())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { @@ -118,3 +217,26 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { _, ok := data["role_level"] return ok } + +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error { + return g.join.IncrVersion(ctx, userID, groupIDs, state) +} + +func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + return g.member.IncrVersion(ctx, groupID, userIDs, state) +} + +func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find member incr version", "groupID", groupID, "version", version) + return g.member.FindChangeLog(ctx, groupID, version, limit) +} + +func (g *GroupMemberMgo) BatchFindMemberIncrVersion(ctx context.Context, groupIDs []string, versions []uint, limits []int) ([]*model.VersionLog, error) { + log.ZDebug(ctx, "Batch find member incr version", "groupIDs", groupIDs, "versions", versions) + return g.member.BatchFindChangeLog(ctx, groupIDs, versions, limits) +} + +func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version) + return g.join.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index 4ae7785276..b1942b7083 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -28,7 +28,7 @@ import ( ) func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { - coll := db.Collection("group_request") + coll := db.Collection(database.GroupRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/log.go b/pkg/common/storage/database/mgo/log.go index 51715bd771..6ff4c60395 100644 --- a/pkg/common/storage/database/mgo/log.go +++ b/pkg/common/storage/database/mgo/log.go @@ -28,7 +28,7 @@ import ( ) func NewLogMongo(db *mongo.Database) (database.Log, error) { - coll := db.Collection("log") + coll := db.Collection(database.LogName) _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ { Keys: bson.D{ diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index a7291fcc8f..fc1fe47eab 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -3,10 +3,12 @@ package mgo import ( "context" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/utils/datautil" - "time" + "golang.org/x/exp/rand" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -93,9 +95,32 @@ func (m *MsgMgo) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDo } func (m *MsgMgo) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { - indexs := make([]int64, 0, len(seqs)) + msgs, err := m.getMsgBySeqIndexIn1Doc(ctx, userID, docID, seqs) + if err != nil { + return nil, err + } + if len(msgs) == len(seqs) { + return msgs, nil + } + tmp := make(map[int64]*model.MsgInfoModel) + for i, val := range msgs { + tmp[val.Msg.Seq] = msgs[i] + } + res := make([]*model.MsgInfoModel, 0, len(seqs)) for _, seq := range seqs { - indexs = append(indexs, m.model.GetMsgIndex(seq)) + if val, ok := tmp[seq]; ok { + res = append(res, val) + } else { + res = append(res, &model.MsgInfoModel{Msg: &model.MsgDataModel{Seq: seq}}) + } + } + return res, nil +} + +func (m *MsgMgo) getMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) { + indexes := make([]int64, 0, len(seqs)) + for _, seq := range seqs { + indexes = append(indexes, m.model.GetMsgIndex(seq)) } pipeline := mongo.Pipeline{ bson.D{{Key: "$match", Value: bson.D{ @@ -106,7 +131,7 @@ func (m *MsgMgo) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID strin {Key: "doc_id", Value: 1}, {Key: "msgs", Value: bson.D{ {Key: "$map", Value: bson.D{ - {Key: "input", Value: indexs}, + {Key: "input", Value: indexes}, {Key: "as", Value: "index"}, {Key: "in", Value: bson.D{ {Key: "$arrayElemAt", Value: bson.A{"$msgs", "$$index"}}, @@ -204,7 +229,7 @@ func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID strin if sort != 1 && sort != -1 { return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1") } - opt := options.Find().SetLimit(1).SetSkip(index).SetSort(bson.M{"doc_id": sort}).SetLimit(1) + opt := options.Find().SetSkip(index).SetSort(bson.M{"_id": sort}).SetLimit(1) filter := bson.M{"doc_id": primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)}} msgs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, filter, opt) if err != nil { @@ -254,125 +279,420 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do return nil } -func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { - where := make(bson.A, 0, 6) +//func (m *MsgMgo) searchCount(ctx context.Context, filter any) (int64, error) { +// +// return nil, nil +//} + +//func (m *MsgMgo) searchMessage(ctx context.Context, filter any, nextID primitive.ObjectID, content bool, limit int) (int64, []*model.MsgInfoModel, primitive.ObjectID, error) { +// var pipeline bson.A +// if !nextID.IsZero() { +// pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) +// } +// pipeline = append(pipeline, +// bson.M{"$match": filter}, +// bson.M{"$limit": limit}, +// bson.M{"$unwind": "$msgs"}, +// bson.M{"$match": filter}, +// bson.M{ +// "$group": bson.M{ +// "_id": "$_id", +// "doc_id": bson.M{ +// "$first": "$doc_id", +// }, +// "msgs": bson.M{"$push": "$msgs"}, +// }, +// }, +// ) +// if !content { +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 1, +// "count": bson.M{"$size": "$msgs"}, +// }, +// }, +// ) +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Count int64 `bson:"count"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, nil +// } +// var count int64 +// for _, r := range res { +// count += r.Count +// } +// return count, nil, res[len(res)-1].ID, nil +// } +// type result struct { +// ID primitive.ObjectID `bson:"_id"` +// Msg []*model.MsgInfoModel `bson:"msgs"` +// } +// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, primitive.ObjectID{}, err +// } +// if len(res) == 0 { +// return 0, nil, primitive.ObjectID{}, err +// } +// var count int +// for _, r := range res { +// count += len(r.Msg) +// } +// msgs := make([]*model.MsgInfoModel, 0, count) +// for _, r := range res { +// msgs = append(msgs, r.Msg...) +// } +// return int64(count), msgs, res[len(res)-1].ID, nil +//} + +/* + +db.msg3.aggregate( + [ + { + "$match": { + "doc_id": "si_7009965934_8710838466:0" + }, + + } + ] +) + + +*/ + +type searchMessageIndex struct { + ID primitive.ObjectID `bson:"_id"` + Index []int64 `bson:"index"` +} + +func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID primitive.ObjectID, limit int) ([]searchMessageIndex, error) { + var pipeline bson.A + if !nextID.IsZero() { + pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) + } + coarseFilter := bson.M{ + "$or": bson.A{ + bson.M{ + "doc_id": primitive.Regex{Pattern: "^sg_"}, + }, + bson.M{ + "doc_id": primitive.Regex{Pattern: "^si_"}, + }, + }, + } + pipeline = append(pipeline, + bson.M{"$sort": bson.M{"_id": 1}}, + bson.M{"$match": coarseFilter}, + bson.M{"$match": filter}, + bson.M{"$limit": limit}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs": bson.M{ + "$map": bson.M{ + "input": "$msgs", + "as": "msg", + "in": bson.M{ + "$mergeObjects": bson.A{ + "$$msg", + bson.M{ + "_search_temp_index": bson.M{ + "$indexOfArray": bson.A{ + "$msgs", "$$msg", + }, + }, + }, + }, + }, + }, + }, + }, + }, + bson.M{"$unwind": "$msgs"}, + bson.M{"$match": filter}, + bson.M{ + "$project": bson.M{ + "_id": 1, + "msgs._search_temp_index": 1, + }, + }, + bson.M{ + "$group": bson.M{ + "_id": "$_id", + "index": bson.M{"$push": "$msgs._search_temp_index"}, + }, + }, + bson.M{"$sort": bson.M{"_id": 1}}, + ) + return mongoutil.Aggregate[searchMessageIndex](ctx, m.coll, pipeline) +} + +func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []searchMessageIndex, error) { + filter := bson.M{} if req.RecvID != "" { - where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) + filter["$or"] = bson.A{ + bson.M{"msgs.msg.recv_id": req.RecvID}, + bson.M{"msgs.msg.group_id": req.RecvID}, + } } if req.SendID != "" { - where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) + filter["msgs.msg.send_id"] = req.SendID } if req.ContentType != 0 { - where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) + filter["msgs.msg.content_type"] = req.ContentType } if req.SessionType != 0 { - where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) + filter["msgs.msg.session_type"] = req.SessionType } if req.SendTime != "" { sendTime, err := time.Parse(time.DateOnly, req.SendTime) if err != nil { return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) } - where = append(where, - bson.M{ - "msgs.msg.send_time": bson.M{ - "$gte": sendTime.UnixMilli(), - }, - }, + filter["$and"] = bson.A{ + bson.M{"msgs.msg.send_time": bson.M{ + "$gte": sendTime.UnixMilli(), + }}, bson.M{ "msgs.msg.send_time": bson.M{ "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), }, }, - ) - } - pipeline := bson.A{ - bson.M{ - "$unwind": "$msgs", - }, - } - if len(where) > 0 { - pipeline = append(pipeline, bson.M{ - "$match": bson.M{"$and": where}, - }) + } } - pipeline = append(pipeline, - bson.M{ - "$project": bson.M{ - "_id": 0, - "msg": "$msgs.msg", - }, - }, - bson.M{ - "$count": "count", - }, + + var ( + nextID primitive.ObjectID + count int + dataRange []searchMessageIndex + skip = int((req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber()) ) - count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) - if err != nil { - return 0, nil, err + _, _ = dataRange, skip + const maxDoc = 50 + data := make([]searchMessageIndex, 0, req.Pagination.GetShowNumber()) + push := cap(data) + for i := 0; ; i++ { + res, err := m.searchMessageIndex(ctx, filter, nextID, maxDoc) + if err != nil { + return 0, nil, err + } + if len(res) > 0 { + nextID = res[len(res)-1].ID + } + for _, r := range res { + var dataIndex []int64 + for _, index := range r.Index { + if push > 0 && count >= skip { + dataIndex = append(dataIndex, index) + push-- + } + count++ + } + if len(dataIndex) > 0 { + data = append(data, searchMessageIndex{ + ID: r.ID, + Index: dataIndex, + }) + } + } + if push <= 0 { + push-- + } + if len(res) < maxDoc || push < -10 { + return int64(count), data, nil + } } - if len(count) == 0 || count[0] == 0 { - return 0, nil, nil +} + +func (m *MsgMgo) getDocRange(ctx context.Context, id primitive.ObjectID, index []int64) ([]*model.MsgInfoModel, error) { + if len(index) == 0 { + return nil, nil + } + + pipeline := bson.A{ + bson.M{"$match": bson.M{"_id": id}}, + bson.M{"$project": "$msgs"}, } - pipeline = pipeline[:len(pipeline)-1] - pipeline = append(pipeline, - bson.M{ - "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), - }, - bson.M{ - "$limit": req.Pagination.GetShowNumber(), - }, - ) msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + return msgs, nil +} + +func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) { + count, data, err := m.searchMessage(ctx, req) if err != nil { return 0, nil, err } - for i := range msgs { - msgInfo := msgs[i] - if msgInfo == nil || msgInfo.Msg == nil { - continue + var msgs []*model.MsgInfoModel + if len(data) > 0 { + var n int + for _, d := range data { + n += len(d.Index) } - if msgInfo.Revoke != nil { - revokeContent := sdkws.MessageRevokedContent{ - RevokerID: msgInfo.Revoke.UserID, - RevokerRole: msgInfo.Revoke.Role, - ClientMsgID: msgInfo.Msg.ClientMsgID, - RevokerNickname: msgInfo.Revoke.Nickname, - RevokeTime: msgInfo.Revoke.Time, - SourceMessageSendTime: msgInfo.Msg.SendTime, - SourceMessageSendID: msgInfo.Msg.SendID, - SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, - SessionType: msgInfo.Msg.SessionType, - Seq: msgInfo.Msg.Seq, - Ex: msgInfo.Msg.Ex, - } - data, err := jsonutil.JsonMarshal(&revokeContent) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") - } - elem := sdkws.NotificationElem{Detail: string(data)} - content, err := jsonutil.JsonMarshal(&elem) - if err != nil { - return 0, nil, errs.WrapMsg(err, "json.Marshal elem") + msgs = make([]*model.MsgInfoModel, 0, n) + } + for _, val := range data { + res, err := mongoutil.FindOne[*model.MsgDocModel](ctx, m.coll, bson.M{"_id": val.ID}) + if err != nil { + return 0, nil, err + } + for _, i := range val.Index { + if i >= int64(len(res.Msg)) { + continue } - msgInfo.Msg.ContentType = constant.MsgRevokeNotification - msgInfo.Msg.Content = string(content) + msgs = append(msgs, res.Msg[i]) } - msgs = append(msgs, msgInfo) - } - //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber - //n := int32(len(msgs)) - //if start >= n { - // return n, []*relation.MsgInfoModel{}, nil - //} - //if start+req.Pagination.ShowNumber < n { - // msgs = msgs[start : start+req.Pagination.ShowNumber] - //} else { - // msgs = msgs[start:] - //} - return count[0], msgs, nil + } + return count, msgs, nil } +//func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { +// where := make(bson.A, 0, 6) +// if req.RecvID != "" { +// if req.SessionType == constant.ReadGroupChatType { +// where = append(where, bson.M{ +// "$or": bson.A{ +// bson.M{"doc_id": "^n_" + req.RecvID + ":"}, +// bson.M{"doc_id": "^sg_" + req.RecvID + ":"}, +// }, +// }) +// } else { +// where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) +// } +// } +// if req.SendID != "" { +// where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) +// } +// if req.ContentType != 0 { +// where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) +// } +// if req.SessionType != 0 { +// where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) +// } +// if req.SendTime != "" { +// sendTime, err := time.Parse(time.DateOnly, req.SendTime) +// if err != nil { +// return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) +// } +// where = append(where, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$gte": sendTime.UnixMilli(), +// }, +// }, +// bson.M{ +// "msgs.msg.send_time": bson.M{ +// "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), +// }, +// }, +// ) +// } +// opt := options.Find().SetLimit(100) +// res, err := mongoutil.Find[model.MsgDocModel](ctx, m.coll, bson.M{"$and": where}, opt) +// if err != nil { +// return 0, nil, err +// } +// _ = res +// fmt.Println() +// +// return 0, nil, nil +// pipeline := bson.A{ +// bson.M{ +// "$unwind": "$msgs", +// }, +// } +// if len(where) > 0 { +// pipeline = append(pipeline, bson.M{ +// "$match": bson.M{"$and": where}, +// }) +// } +// pipeline = append(pipeline, +// bson.M{ +// "$project": bson.M{ +// "_id": 0, +// "msg": "$msgs.msg", +// }, +// }, +// bson.M{ +// "$count": "count", +// }, +// ) +// //count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) +// //if err != nil { +// // return 0, nil, err +// //} +// //if len(count) == 0 || count[0] == 0 { +// // return 0, nil, nil +// //} +// count := []int32{0} +// pipeline = pipeline[:len(pipeline)-1] +// pipeline = append(pipeline, +// bson.M{ +// "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), +// }, +// bson.M{ +// "$limit": req.Pagination.GetShowNumber(), +// }, +// ) +// msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) +// if err != nil { +// return 0, nil, err +// } +// for i := range msgs { +// msgInfo := msgs[i] +// if msgInfo == nil || msgInfo.Msg == nil { +// continue +// } +// if msgInfo.Revoke != nil { +// revokeContent := sdkws.MessageRevokedContent{ +// RevokerID: msgInfo.Revoke.UserID, +// RevokerRole: msgInfo.Revoke.Role, +// ClientMsgID: msgInfo.Msg.ClientMsgID, +// RevokerNickname: msgInfo.Revoke.Nickname, +// RevokeTime: msgInfo.Revoke.Time, +// SourceMessageSendTime: msgInfo.Msg.SendTime, +// SourceMessageSendID: msgInfo.Msg.SendID, +// SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, +// SessionType: msgInfo.Msg.SessionType, +// Seq: msgInfo.Msg.Seq, +// Ex: msgInfo.Msg.Ex, +// } +// data, err := jsonutil.JsonMarshal(&revokeContent) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") +// } +// elem := sdkws.NotificationElem{Detail: string(data)} +// content, err := jsonutil.JsonMarshal(&elem) +// if err != nil { +// return 0, nil, errs.WrapMsg(err, "json.Marshal elem") +// } +// msgInfo.Msg.ContentType = constant.MsgRevokeNotification +// msgInfo.Msg.Content = string(content) +// } +// } +// //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber +// //n := int32(len(msgs)) +// //if start >= n { +// // return n, []*relation.MsgInfoModel{}, nil +// //} +// //if start+req.Pagination.ShowNumber < n { +// // msgs = msgs[start : start+req.Pagination.ShowNumber] +// //} else { +// // msgs = msgs[start:] +// //} +// return count[0], msgs, nil +//} + func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { var sort int if ase { @@ -907,10 +1227,53 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { + limit := 5000 + var skip int + var docIDs []string + var offset int + + count, err := m.coll.CountDocuments(ctx, bson.M{}) + if err != nil { + return nil, err + } + + if count < int64(limit) { + skip = 0 + } else { + rand.Seed(uint64(time.Now().UnixMilli())) + skip = rand.Intn(int(count / int64(limit))) + offset = skip * limit + } + log.ZDebug(ctx, "offset", "skip", skip, "offset", offset) + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ + { + "$project": bson.M{ + "doc_id": 1, + }, + }, + { + "$skip": offset, + }, + { + "$limit": limit, + }, + }) + + for _, doc := range res { + docIDs = append(docIDs, doc.DocID) + } + + return docIDs, errs.Wrap(err) +} + +func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ + "doc_id": bson.M{ + "$in": docIDs, + }, "msgs.msg.send_time": bson.M{ "$lt": ts, }, diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go new file mode 100644 index 0000000000..5aed4dc511 --- /dev/null +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -0,0 +1,75 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "math/rand" + "strconv" + "testing" + "time" +) + +func TestName1(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + + req := &msg.SearchMessageReq{ + //RecvID: "3187706596", + //SendID: "7009965934", + ContentType: 101, + //SendTime: "2024-05-06", + //SessionType: 3, + Pagination: &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 10, + }, + } + total, res, err := v.SearchMessage(ctx, req) + if err != nil { + panic(err) + } + + for i, re := range res { + t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + } + + t.Log(total) +} + +func TestName10(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + v := &MsgMgo{ + coll: cli.Database("openim_v3").Collection("msg3"), + } + opt := options.Find().SetLimit(1000) + + res, err := mongoutil.Find[model.MsgDocModel](ctx, v.coll, bson.M{}, opt) + if err != nil { + panic(err) + } + ctx = context.Background() + for i := 0; i < 100000; i++ { + for j := range res { + res[j].DocID = strconv.FormatUint(rand.Uint64(), 10) + ":0" + } + if err := mongoutil.InsertMany(ctx, v.coll, res); err != nil { + panic(err) + } + t.Log("====>", time.Now(), i) + } + +} diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 8ed7b3a561..4242fbb53a 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -16,10 +16,13 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -27,7 +30,7 @@ import ( ) func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { - coll := db.Collection("s3") + coll := db.Collection(database.ObjectName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "name", Value: 1}, @@ -68,3 +71,14 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ + "create_time": bson.M{"$lt": duration}, + }, pagination) +} +func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{ + "key": key, + "create_time": bson.M{"$gt": duration}, + }) +} diff --git a/pkg/common/storage/database/mgo/seq_conversation.go b/pkg/common/storage/database/mgo/seq_conversation.go new file mode 100644 index 0000000000..7971b7e1a7 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -0,0 +1,103 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqConversationMongo{coll: coll}, nil +} + +type seqConversationMongo struct { + coll *mongo.Collection +} + +func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "conversation_id": conversationID, + } + insert := bson.M{ + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errors.New("size must be greater than 0") + } + if size == 0 { + return s.GetMaxSeq(ctx, conversationID) + } + filter := map[string]any{"conversation_id": conversationID} + update := map[string]any{ + "$inc": map[string]any{"max_seq": size}, + "$set": map[string]any{"min_seq": int64(0)}, + } + opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) + lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) + if err != nil { + return 0, err + } + return lastSeq - size, nil +} + +func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "max_seq") +} + +func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "min_seq") +} + +func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { + return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID}) +} diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go new file mode 100644 index 0000000000..42507a6937 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -0,0 +1,42 @@ +package mgo + +import ( + "context" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Mongodb() *mongo.Database { + return Result( + mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)), + ).Database("openim_v3") +} + +func TestUserSeq(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.SetUserMinSeq(context.Background(), "1000", "2000", 4)) +} + +func TestConversationSeq(t *testing.T) { + cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo) + t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10)) + t.Log(cSeq.Malloc(context.Background(), "2000", 10)) + t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) +} + +func TestUserGetUserReadSeqs(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.GetUserReadSeqs(context.Background(), "2110910952", []string{"sg_345762580", "2000", "3000"})) +} diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go new file mode 100644 index 0000000000..244de30000 --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -0,0 +1,126 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { + coll := db.Collection(database.SeqUserName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "user_id", Value: 1}, + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqUserMongo{coll: coll}, nil +} + +type seqUserMongo struct { + coll *mongo.Collection +} + +func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + insert := bson.M{ + "user_id": userID, + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + "read_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1}) + seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt) + if err == nil { + return seq, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return 0, nil + } else { + return 0, err + } +} + +func (s *seqUserMongo) GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "max_seq") +} + +func (s *seqUserMongo) SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "max_seq") +} + +func (s *seqUserMongo) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "min_seq") +} + +func (s *seqUserMongo) SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "min_seq") +} + +func (s *seqUserMongo) GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "read_seq") +} + +func (s *seqUserMongo) notFoundSet0(seq map[string]int64, conversationIDs []string) { + for _, conversationID := range conversationIDs { + if _, ok := seq[conversationID]; !ok { + seq[conversationID] = 0 + } + } +} + +func (s *seqUserMongo) GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) { + if len(conversationID) == 0 { + return map[string]int64{}, nil + } + filter := bson.M{"user_id": userID, "conversation_id": bson.M{"$in": conversationID}} + opt := options.Find().SetProjection(bson.M{"_id": 0, "conversation_id": 1, "read_seq": 1}) + seqs, err := mongoutil.Find[*model.SeqUser](ctx, s.coll, filter, opt) + if err != nil { + return nil, err + } + res := make(map[string]int64) + for _, seq := range seqs { + res[seq.ConversationID] = seq.ReadSeq + } + s.notFoundSet0(res, conversationID) + return res, nil +} + +func (s *seqUserMongo) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) + if err != nil { + return err + } + if dbSeq > seq { + return nil + } + return s.setSeq(ctx, conversationID, userID, seq, "read_seq") +} diff --git a/pkg/common/storage/database/mgo/subscribe.go b/pkg/common/storage/database/mgo/subscribe.go deleted file mode 100644 index 5b7d9786b7..0000000000 --- a/pkg/common/storage/database/mgo/subscribe.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - - "github.com/openimsdk/tools/errs" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -// prefixes and suffixes. -const ( - SubscriptionPrefix = "subscription_prefix" - SubscribedPrefix = "subscribed_prefix" -) - -// MaximumSubscription Maximum number of subscriptions. -const ( - MaximumSubscription = 3000 -) - -func NewUserMongoDriver(database *mongo.Database) database.SubscribeUser { - return &UserMongoDriver{ - userCollection: database.Collection(model.SubscribeUserTableName), - } -} - -type UserMongoDriver struct { - userCollection *mongo.Collection -} - -// AddSubscriptionList Subscriber's handling of thresholds. -func (u *UserMongoDriver) AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error { - // Check the number of lists in the key. - pipeline := mongo.Pipeline{ - {{"$match", bson.D{{"user_id", SubscriptionPrefix + userID}}}}, - {{"$project", bson.D{{"count", bson.D{{"$size", "$user_id_list"}}}}}}, - } - // perform aggregate operations - cursor, err := u.userCollection.Aggregate(ctx, pipeline) - if err != nil { - return errs.Wrap(err) - } - defer cursor.Close(ctx) - var cnt struct { - Count int `bson:"count"` - } - // iterate over aggregated results - for cursor.Next(ctx) { - err = cursor.Decode(&cnt) - if err != nil { - return errs.Wrap(err) - } - } - var newUserIDList []string - // If the threshold is exceeded, pop out the previous MaximumSubscription - len(userIDList) and insert it. - if cnt.Count+len(userIDList) > MaximumSubscription { - newUserIDList, err = u.GetAllSubscribeList(ctx, userID) - if err != nil { - return err - } - newUserIDList = newUserIDList[MaximumSubscription-len(userIDList):] - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$set": bson.M{"user_id_list": newUserIDList}}, - ) - if err != nil { - return err - } - // Another way to subscribe to N before pop,Delete after testing - /*for i := 1; i <= MaximumSubscription-len(userIDList); i++ { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{SubscriptionPrefix + userID: bson.M{"$pop": -1}}, - ) - if err != nil { - return err - } - }*/ - } - upsert := true - opts := &options.UpdateOptions{ - Upsert: &upsert, - } - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$addToSet": bson.M{"user_id_list": bson.M{"$each": userIDList}}}, - opts, - ) - if err != nil { - return errs.Wrap(err) - } - for _, user := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + user}, - bson.M{"$addToSet": bson.M{"user_id_list": userID}}, - opts, - ) - if err != nil { - return errs.WrapMsg(err, "transaction failed") - } - } - return nil -} - -// UnsubscriptionList Handling of unsubscribe. -func (u *UserMongoDriver) UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error { - _, err := u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}, - bson.M{"$pull": bson.M{"user_id_list": bson.M{"$in": userIDList}}}, - ) - if err != nil { - return errs.Wrap(err) - } - err = u.RemoveSubscribedListFromUser(ctx, userID, userIDList) - if err != nil { - return errs.Wrap(err) - } - return nil -} - -// RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. -func (u *UserMongoDriver) RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error { - var err error - for _, userIDTemp := range userIDList { - _, err = u.userCollection.UpdateOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userIDTemp}, - bson.M{"$pull": bson.M{"user_id_list": userID}}, - ) - } - return errs.Wrap(err) -} - -// GetAllSubscribeList Get all users subscribed by this user. -func (u *UserMongoDriver) GetAllSubscribeList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscriptionPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} - -// GetSubscribedList Get the user subscribed by those users. -func (u *UserMongoDriver) GetSubscribedList(ctx context.Context, userID string) (userIDList []string, err error) { - var user model.SubscribeUser - cursor := u.userCollection.FindOne( - ctx, - bson.M{"user_id": SubscribedPrefix + userID}) - err = cursor.Decode(&user) - if err != nil { - if err == mongo.ErrNoDocuments { - return []string{}, nil - } else { - return nil, errs.Wrap(err) - } - } - return user.UserIDList, nil -} diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 96cb18882b..ee92b75544 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -31,7 +31,7 @@ import ( ) func NewUserMongo(db *mongo.Database) (database.User, error) { - coll := db.Collection("user") + coll := db.Collection(database.UserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, @@ -167,6 +167,10 @@ func (u *UserMgo) DeleteUserCommand(ctx context.Context, userID string, Type int filter := bson.M{"userID": userID, "type": Type, "uuid": UUID} result, err := collection.DeleteOne(ctx, filter) + // when err is not nil, result might be nil + if err != nil { + return errs.Wrap(err) + } if result.DeletedCount == 0 { // No records found to update return errs.Wrap(errs.ErrRecordNotFound) @@ -319,3 +323,69 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (u *UserMgo) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + if len(userIDName) == 0 { + return nil, nil + } + userIDs := make([]string, 0, len(userIDName)) + attached := make(map[string]string) + for userID, name := range userIDName { + userIDs = append(userIDs, userID) + if name == "" { + continue + } + attached[userID] = name + } + var sortValue int + if asc { + sortValue = 1 + } else { + sortValue = -1 + } + if len(attached) == 0 { + filter := bson.M{"user_id": bson.M{"$in": userIDs}} + opt := options.Find().SetSort(bson.M{"nickname": sortValue}) + return mongoutil.Find[*model.User](ctx, u.coll, filter, opt) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": bson.M{"$in": userIDs}, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$arrayElemAt": []any{ + bson.M{ + "$filter": bson.M{ + "input": bson.M{ + "$objectToArray": attached, + }, + "as": "item", + "cond": bson.M{ + "$eq": []any{"$$item.k", "$user_id"}, + }, + }, + }, + 0, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$ifNull": []any{"$_query_sort_name.v", "$nickname"}, + }, + }, + }, + { + "$sort": bson.M{ + "_query_sort_name": sortValue, + }, + }, + } + return mongoutil.Aggregate[*model.User](ctx, u.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go new file mode 100644 index 0000000000..2c4bdef4e8 --- /dev/null +++ b/pkg/common/storage/database/mgo/version_log.go @@ -0,0 +1,304 @@ +package mgo + +import ( + "context" + "errors" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { + lm := &VersionLogMgo{coll: coll} + if err := lm.initIndex(context.Background()); err != nil { + return nil, errs.WrapMsg(err, "init version log index failed", "coll", coll.Name()) + } + return lm, nil +} + +type VersionLogMgo struct { + coll *mongo.Collection +} + +func (l *VersionLogMgo) initIndex(ctx context.Context) error { + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + Options: options.Index().SetUnique(true), + }) + + return err +} + +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { + _, err := l.IncrVersionResult(ctx, dId, eIds, state) + return err +} + +func (l *VersionLogMgo) IncrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + vl, err := l.incrVersionResult(ctx, dId, eIds, state) + if err != nil { + return nil, err + } + versionctx.GetVersionLog(ctx).Append(versionctx.Collection{ + Name: l.coll.Name(), + Doc: vl, + }) + return vl, nil +} + +func (l *VersionLogMgo) incrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + now := time.Now() + if res, err := l.writeLogBatch2(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + if res, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !mongo.IsDuplicateKeyError(err) { + return nil, err + } + return l.writeLogBatch2(ctx, dId, eIds, state, now) +} + +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { + wl := model.VersionLogTable{ + ID: primitive.NewObjectID(), + DID: dId, + Logs: make([]model.VersionLogElem, 0, len(eIds)), + Version: database.FirstVersion, + Deleted: database.DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, model.VersionLogElem{ + EID: eId, + State: state, + Version: database.FirstVersion, + LastUpdate: now, + }) + } + if _, err := l.coll.InsertOne(ctx, &wl); err != nil { + return nil, err + } + return wl.VersionLog(), nil +} + +func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { + if eIds == nil { + eIds = []string{} + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "state": state, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + projection := bson.M{ + "logs": 0, + } + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(projection) + res, err := mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) + if err != nil { + return nil, err + } + res.Logs = make([]model.VersionLogElem, 0, len(eIds)) + for _, id := range eIds { + res.Logs = append(res.Logs, model.VersionLogElem{ + EID: id, + State: state, + Version: res.Version, + LastUpdate: res.LastUpdate, + }) + } + return res, nil +} + +func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { + vl, err := mongoutil.FindOne[*model.VersionLogTable](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err != nil { + return nil, err + } + return vl.VersionLog(), nil +} + +func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { + if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { + return wl, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + log.ZDebug(ctx, "init doc", "dId", dId) + if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dId) + return res, nil + } else if mongo.IsDuplicateKeyError(err) { + return l.findChangeLog(ctx, dId, version, limit) + } else { + return nil, err + } +} + +func (l *VersionLogMgo) BatchFindChangeLog(ctx context.Context, dIds []string, versions []uint, limits []int) (vLogs []*model.VersionLog, err error) { + for i := 0; i < len(dIds); i++ { + if vLog, err := l.findChangeLog(ctx, dIds[i], versions[i], limits[i]); err == nil { + vLogs = append(vLogs, vLog) + } else if !errors.Is(err, mongo.ErrNoDocuments) { + log.ZError(ctx, "findChangeLog error:", errs.Wrap(err)) + } + log.ZDebug(ctx, "init doc", "dId", dIds[i]) + if res, err := l.initDoc(ctx, dIds[i], nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dIds[i]) + vLogs = append(vLogs, res) + } else if mongo.IsDuplicateKeyError(err) { + l.findChangeLog(ctx, dIds[i], versions[i], limits[i]) + } else { + log.ZError(ctx, "init doc error:", errs.Wrap(err)) + } + } + return vLogs, errs.Wrap(err) +} + +func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { + if version == 0 && limit == 0 { + return l.findDoc(ctx, dId) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": dId, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + vl, err := mongoutil.Aggregate[*model.VersionLog](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(vl) == 0 { + return nil, mongo.ErrNoDocuments + } + return vl[0], nil +} + +func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) +} + +func (l *VersionLogMgo) Delete(ctx context.Context, dId string) error { + return mongoutil.DeleteOne(ctx, l.coll, bson.M{"d_id": dId}) +} diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go new file mode 100644 index 0000000000..4576e45bcd --- /dev/null +++ b/pkg/common/storage/database/mgo/version_test.go @@ -0,0 +1,39 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +//func Result[V any](val V, err error) V { +// if err != nil { +// panic(err) +// } +// return val +//} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("version_test") + tmp, err := NewVersionLog(coll) + if err != nil { + panic(err) + } + vl := tmp.(*VersionLogMgo) + res, err := vl.incrVersionResult(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert) + if err != nil { + t.Log(err) + return + } + t.Logf("%+v", res) +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b402f3ac7d..23a99f5b96 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -16,10 +16,11 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/msg" "go.mongodb.org/mongo-driver/mongo" - "time" ) type Msg interface { @@ -37,12 +38,14 @@ type Msg interface { GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error - SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) + SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) + GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + + GetDocIDs(ctx context.Context) ([]string, error) } diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go new file mode 100644 index 0000000000..748bd844d2 --- /dev/null +++ b/pkg/common/storage/database/name.go @@ -0,0 +1,20 @@ +package database + +const ( + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + ConversationVersionName = "conversation_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" + SeqConversationName = "seq" + SeqUserName = "seq_user" +) diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 554f71f35d..8292006a04 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -16,11 +16,16 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go new file mode 100644 index 0000000000..cf93b795f4 --- /dev/null +++ b/pkg/common/storage/database/seq.go @@ -0,0 +1,11 @@ +package database + +import "context" + +type SeqConversation interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go new file mode 100644 index 0000000000..9f75c710b6 --- /dev/null +++ b/pkg/common/storage/database/seq_user.go @@ -0,0 +1,13 @@ +package database + +import "context" + +type SeqUser interface { + GetUserMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetUserReadSeqs(ctx context.Context, userID string, conversationID []string) (map[string]int64, error) +} diff --git a/pkg/common/storage/database/subscribe.go b/pkg/common/storage/database/subscribe.go deleted file mode 100644 index 5905ecd073..0000000000 --- a/pkg/common/storage/database/subscribe.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Β© 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import "context" - -// SubscribeUser Operation interface of user mongodb. -type SubscribeUser interface { - // AddSubscriptionList Subscriber's handling of thresholds. - AddSubscriptionList(ctx context.Context, userID string, userIDList []string) error - // UnsubscriptionList Handling of unsubscribe. - UnsubscriptionList(ctx context.Context, userID string, userIDList []string) error - // RemoveSubscribedListFromUser Among the unsubscribed users, delete the user from the subscribed list. - RemoveSubscribedListFromUser(ctx context.Context, userID string, userIDList []string) error - // GetAllSubscribeList Get all users subscribed by this user - GetAllSubscribeList(ctx context.Context, id string) (userIDList []string, err error) - // GetSubscribedList Get the user subscribed by those users - GetSubscribedList(ctx context.Context, id string) (userIDList []string, err error) -} diff --git a/pkg/common/storage/database/user.go b/pkg/common/storage/database/user.go index 2e40886205..4ddc8285f6 100644 --- a/pkg/common/storage/database/user.go +++ b/pkg/common/storage/database/user.go @@ -39,6 +39,9 @@ type User interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get user total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) + // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go new file mode 100644 index 0000000000..28224a7c79 --- /dev/null +++ b/pkg/common/storage/database/version_log.go @@ -0,0 +1,21 @@ +package database + +import ( + "context" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type VersionLog interface { + IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + BatchFindChangeLog(ctx context.Context, dIds []string, versions []uint, limits []int) ([]*model.VersionLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error + Delete(ctx context.Context, dId string) error +} diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2a..abcca2f2b9 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -15,17 +15,19 @@ package model import ( + "go.mongodb.org/mongo-driver/bson/primitive" "time" ) // Friend represents the data structure for a friend relationship in MongoDB. type Friend struct { - OwnerUserID string `bson:"owner_user_id"` - FriendUserID string `bson:"friend_user_id"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` + ID primitive.ObjectID `bson:"_id"` + OwnerUserID string `bson:"owner_user_id"` + FriendUserID string `bson:"friend_user_id"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 8095665d2f..e16233973b 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -92,15 +92,15 @@ type GroupCount struct { Count int64 `bson:"count"` } -func (MsgDocModel) TableName() string { +func (*MsgDocModel) TableName() string { return MsgTableName } -func (MsgDocModel) GetSingleGocMsgNum() int64 { +func (*MsgDocModel) GetSingleGocMsgNum() int64 { return singleGocMsgNum } -func (MsgDocModel) GetSingleGocMsgNum5000() int64 { +func (*MsgDocModel) GetSingleGocMsgNum5000() int64 { return singleGocMsgNum5000 } @@ -108,12 +108,12 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } -func (m MsgDocModel) GetDocID(conversationID string, seq int64) string { +func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) } -func (m MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 { +func (m *MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 { t := make(map[string][]int64) for i := 0; i < len(seqs); i++ { docID := m.GetDocID(conversationID, seqs[i]) @@ -127,15 +127,15 @@ func (m MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[st return t } -func (MsgDocModel) GetMsgIndex(seq int64) int64 { +func (*MsgDocModel) GetMsgIndex(seq int64) int64 { return (seq - 1) % singleGocMsgNum } -func (MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { +func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } -func (MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { +func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) msgModel.Seq = v diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go new file mode 100644 index 0000000000..1dc75eff18 --- /dev/null +++ b/pkg/common/storage/model/seq.go @@ -0,0 +1,7 @@ +package model + +type SeqConversation struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} diff --git a/pkg/common/storage/model/seq_user.go b/pkg/common/storage/model/seq_user.go new file mode 100644 index 0000000000..845996bb8b --- /dev/null +++ b/pkg/common/storage/model/seq_user.go @@ -0,0 +1,9 @@ +package model + +type SeqUser struct { + UserID string `bson:"user_id"` + ConversationID string `bson:"conversation_id"` + MinSeq int64 `bson:"min_seq"` + MaxSeq int64 `bson:"max_seq"` + ReadSeq int64 `bson:"read_seq"` +} diff --git a/pkg/common/storage/model/user.go b/pkg/common/storage/model/user.go index c6a4f952c2..f64d09e797 100644 --- a/pkg/common/storage/model/user.go +++ b/pkg/common/storage/model/user.go @@ -36,10 +36,10 @@ func (u *User) GetFaceURL() string { return u.FaceURL } -func (u User) GetUserID() string { +func (u *User) GetUserID() string { return u.UserID } -func (u User) GetEx() string { +func (u *User) GetEx() string { return u.Ex } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go new file mode 100644 index 0000000000..6ed8d30f2e --- /dev/null +++ b/pkg/common/storage/model/version_log.go @@ -0,0 +1,74 @@ +package model + +import ( + "context" + "errors" + "github.com/openimsdk/tools/log" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +const ( + VersionStateInsert = iota + 1 + VersionStateDelete + VersionStateUpdate +) + +const ( + VersionGroupChangeID = "" + VersionSortChangeID = "____S_O_R_T_I_D____" +) + +type VersionLogElem struct { + EID string `bson:"e_id"` + State int32 `bson:"state"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type VersionLogTable struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (v *VersionLogTable) VersionLog() *VersionLog { + return &VersionLog{ + ID: v.ID, + DID: v.DID, + Logs: v.Logs, + Version: v.Version, + Deleted: v.Deleted, + LastUpdate: v.LastUpdate, + LogLen: len(v.Logs), + } +} + +type VersionLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` +} + +func (v *VersionLog) DeleteAndChangeIDs() (insertIds, deleteIds, updateIds []string) { + for _, l := range v.Logs { + switch l.State { + case VersionStateInsert: + insertIds = append(insertIds, l.EID) + case VersionStateDelete: + deleteIds = append(deleteIds, l.EID) + case VersionStateUpdate: + updateIds = append(updateIds, l.EID) + default: + log.ZError(context.Background(), "invalid version status found", errors.New("dirty database data"), "objID", v.ID.Hex(), "did", v.DID, "elem", l) + } + } + return +} diff --git a/pkg/common/storage/versionctx/rpc.go b/pkg/common/storage/versionctx/rpc.go new file mode 100644 index 0000000000..67b95aebd5 --- /dev/null +++ b/pkg/common/storage/versionctx/rpc.go @@ -0,0 +1,14 @@ +package versionctx + +import ( + "context" + "google.golang.org/grpc" +) + +func EnableVersionCtx() grpc.ServerOption { + return grpc.ChainUnaryInterceptor(enableVersionCtxInterceptor) +} + +func enableVersionCtxInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(WithVersionLog(ctx), req) +} diff --git a/pkg/common/storage/versionctx/version.go b/pkg/common/storage/versionctx/version.go new file mode 100644 index 0000000000..5db8856401 --- /dev/null +++ b/pkg/common/storage/versionctx/version.go @@ -0,0 +1,48 @@ +package versionctx + +import ( + "context" + tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "sync" +) + +type Collection struct { + Name string + Doc *tablerelation.VersionLog +} + +type versionKey struct{} + +func WithVersionLog(ctx context.Context) context.Context { + return context.WithValue(ctx, versionKey{}, &VersionLog{}) +} + +func GetVersionLog(ctx context.Context) *VersionLog { + if v, ok := ctx.Value(versionKey{}).(*VersionLog); ok { + return v + } + return nil +} + +type VersionLog struct { + lock sync.Mutex + data []Collection +} + +func (v *VersionLog) Append(data ...Collection) { + if v == nil || len(data) == 0 { + return + } + v.lock.Lock() + defer v.lock.Unlock() + v.data = append(v.data, data...) +} + +func (v *VersionLog) Get() []Collection { + if v == nil { + return nil + } + v.lock.Lock() + defer v.lock.Unlock() + return v.data +} diff --git a/pkg/localcache/cache.go b/pkg/localcache/cache.go index 0e040ad389..ba849f8926 100644 --- a/pkg/localcache/cache.go +++ b/pkg/localcache/cache.go @@ -31,6 +31,12 @@ type Cache[V any] interface { Stop() } +func LRUStringHash(key string) uint64 { + h := fnv.New64a() + h.Write(*(*[]byte)(unsafe.Pointer(&key))) + return h.Sum64() +} + func New[V any](opts ...Option) Cache[V] { opt := defaultOption() for _, o := range opts { @@ -49,11 +55,7 @@ func New[V any](opts ...Option) Cache[V] { if opt.localSlotNum == 1 { c.local = createSimpleLRU() } else { - c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, func(key string) uint64 { - h := fnv.New64a() - h.Write(*(*[]byte)(unsafe.Pointer(&key))) - return h.Sum64() - }, createSimpleLRU) + c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, LRUStringHash, createSimpleLRU) } if opt.linkSlotNum > 0 { c.link = link.New(opt.linkSlotNum) diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 64280f238f..726535c48c 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,6 +20,9 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + Set(key K, value V) + SetHas(key K, value V) bool + GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index 970ac083e3..df6bacbf43 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -51,6 +51,11 @@ type ExpirationLRU[K comparable, V any] struct { target Target } +func (x *ExpirationLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + //TODO implement me + panic("implement me") +} + func (x *ExpirationLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { x.lock.Lock() v, ok := x.core.Get(key) @@ -89,5 +94,21 @@ func (x *ExpirationLRU[K, V]) Del(key K) bool { return ok } +func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &expirationLruItem[V]{value: value}) + return true + } + return false +} + +func (x *ExpirationLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &expirationLruItem[V]{value: value}) +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index d6e64aae43..e7f7b8bd51 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,6 +88,86 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } +func (x *LayLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + err error + once sync.Once + ) + + x.lock.Lock() + res := make(map[K]V) + queries := make([]K, 0) + setVs := make(map[K]*layLruItem[V]) + for _, key := range keys { + v, ok := x.core.Get(key) + if ok { + x.lock.Unlock() + v.lock.Lock() + expires, value, err1 := v.expires, v.value, v.err + if expires != 0 && expires > time.Now().UnixMilli() { + v.lock.Unlock() + x.target.IncrGetHit() + res[key] = value + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + continue + } + } + queries = append(queries, key) + x.lock.Unlock() + } + values, err1 := fetch(queries) + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + for key, val := range values { + v := &layLruItem[V]{} + v.value = val + + if err == nil { + v.expires = time.Now().Add(x.successTTL).UnixMilli() + x.target.IncrGetSuccess() + } else { + v.expires = time.Now().Add(x.failedTTL).UnixMilli() + x.target.IncrGetFailed() + } + setVs[key] = v + x.lock.Lock() + x.core.Add(key, v) + x.lock.Unlock() + res[key] = val + } + + return res, err +} + +//func (x *LayLRU[K, V]) Has(key K) bool { +// x.lock.Lock() +// defer x.lock.Unlock() +// return x.core.Contains(key) +//} + +func (x *LayLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +} + +func (x *LayLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) + return true + } + return false +} + func (x *LayLRU[K, V]) Del(key K) bool { x.lock.Lock() ok := x.core.Remove(key) diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index d034e94d32..077219b75f 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -32,6 +32,29 @@ type slotLRU[K comparable, V any] struct { hash func(k K) uint64 } +func (x *slotLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + slotKeys = make(map[uint64][]K) + vs = make(map[K]V) + ) + + for _, k := range keys { + index := x.getIndex(k) + slotKeys[index] = append(slotKeys[index], k) + } + + for k, v := range slotKeys { + batches, err := x.slots[k].GetBatch(v, fetch) + if err != nil { + return nil, err + } + for key, value := range batches { + vs[key] = value + } + } + return vs, nil +} + func (x *slotLRU[K, V]) getIndex(k K) uint64 { return x.hash(k) % x.n } @@ -40,6 +63,14 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) Set(key K, value V) { + x.slots[x.getIndex(key)].Set(key, value) +} + +func (x *slotLRU[K, V]) SetHas(key K, value V) bool { + return x.slots[x.getIndex(key)].SetHas(key, value) +} + func (x *slotLRU[K, V]) Del(key K) bool { return x.slots[x.getIndex(key)].Del(key) } diff --git a/pkg/localcache/option.go b/pkg/localcache/option.go index 00bb9d0441..7d91aba6c3 100644 --- a/pkg/localcache/option.go +++ b/pkg/localcache/option.go @@ -30,7 +30,7 @@ func defaultOption() *option { localSuccessTTL: time.Minute, localFailedTTL: time.Second * 5, delFn: make([]func(ctx context.Context, key ...string), 0, 2), - target: emptyTarget{}, + target: EmptyTarget{}, } } @@ -123,14 +123,14 @@ func WithDeleteKeyBefore(fn func(ctx context.Context, key ...string)) Option { } } -type emptyTarget struct{} +type EmptyTarget struct{} -func (e emptyTarget) IncrGetHit() {} +func (e EmptyTarget) IncrGetHit() {} -func (e emptyTarget) IncrGetSuccess() {} +func (e EmptyTarget) IncrGetSuccess() {} -func (e emptyTarget) IncrGetFailed() {} +func (e EmptyTarget) IncrGetFailed() {} -func (e emptyTarget) IncrDelHit() {} +func (e EmptyTarget) IncrDelHit() {} -func (e emptyTarget) IncrDelNotFound() {} +func (e EmptyTarget) IncrDelNotFound() {} diff --git a/pkg/msgprocessor/conversation.go b/pkg/msgprocessor/conversation.go index b369269cc8..f8140cc7df 100644 --- a/pkg/msgprocessor/conversation.go +++ b/pkg/msgprocessor/conversation.go @@ -24,6 +24,10 @@ import ( "google.golang.org/protobuf/proto" ) +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "g_") || strings.HasPrefix(conversationID, "sg_") +} + func GetNotificationConversationIDByMsg(msg *sdkws.MsgData) string { switch msg.SessionType { case constant.SingleChatType: diff --git a/pkg/rpccache/common.go b/pkg/rpccache/common.go index 5ed007edc4..15b3a8e094 100644 --- a/pkg/rpccache/common.go +++ b/pkg/rpccache/common.go @@ -14,6 +14,11 @@ package rpccache +import ( + "github.com/openimsdk/tools/errs" + "google.golang.org/protobuf/proto" +) + func newListMap[V comparable](values []V, err error) (*listMap[V], error) { if err != nil { return nil, err @@ -32,3 +37,41 @@ type listMap[V comparable] struct { List []V Map map[V]struct{} } + +func respProtoMarshal(resp proto.Message, err error) ([]byte, error) { + if err != nil { + return nil, err + } + return proto.Marshal(resp) +} + +func cacheUnmarshal[V any](resp []byte, err error) (*V, error) { + if err != nil { + return nil, err + } + var val V + if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil { + return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error") + } + return &val, nil +} + +type cacheProto[V any] struct{} + +func (cacheProto[V]) Marshal(resp *V, err error) ([]byte, error) { + if err != nil { + return nil, err + } + return proto.Marshal(any(resp).(proto.Message)) +} + +func (cacheProto[V]) Unmarshal(resp []byte, err error) (*V, error) { + if err != nil { + return nil, err + } + var val V + if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil { + return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error") + } + return &val, nil +} diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 4c00dd1f7e..925d2a37ca 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -16,15 +16,20 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" + "golang.org/x/sync/errgroup" +) + +const ( + conversationWorkerCount = 20 ) func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { @@ -32,7 +37,7 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &ConversationLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -48,21 +53,30 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach type ConversationLocalCache struct { client rpcclient.ConversationRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs req", "ownerUserID", ownerUserID) + resp, err := c.getConversationIDs(ctx, ownerUserID) + if err != nil { + return nil, err + } + return resp.ConversationIDs, nil +} + +func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconversation.GetConversationIDsResp, err error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs req", "ownerUserID", ownerUserID) defer func() { if err == nil { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs return", "value", val) + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs return", "ownerUserID", ownerUserID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversationIDs return", err) + log.ZError(ctx, "ConversationLocalCache getConversationIDs return", err, "ownerUserID", ownerUserID) } }() - return localcache.AnyValue[[]string](c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) (any, error) { - log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs rpc", "ownerUserID", ownerUserID) - return c.client.GetConversationIDs(ctx, ownerUserID) + var cache cacheProto[pbconversation.GetConversationIDsResp] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationIDs rpc", "ownerUserID", ownerUserID) + return cache.Marshal(c.client.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID})) })) } @@ -70,14 +84,15 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID) defer func() { if err == nil { - log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "value", val) + log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "userID", userID, "conversationID", conversationID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversation return", err) + log.ZWarn(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() - return localcache.AnyValue[*pbconversation.Conversation](c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) (any, error) { + var cache cacheProto[pbconversation.Conversation] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID) - return c.client.GetConversation(ctx, userID, conversationID) + return cache.Marshal(c.client.GetConversation(ctx, userID, conversationID)) })) } @@ -90,23 +105,51 @@ func (c *ConversationLocalCache) GetSingleConversationRecvMsgOpt(ctx context.Con } func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { - conversations := make([]*pbconversation.Conversation, 0, len(conversationIDs)) + var ( + conversations = make([]*pbconversation.Conversation, 0, len(conversationIDs)) + conversationsChan = make(chan *pbconversation.Conversation, len(conversationIDs)) + ) + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(conversationWorkerCount) + for _, conversationID := range conversationIDs { - conversation, err := c.GetConversation(ctx, ownerUserID, conversationID) - if err != nil { - if errs.ErrRecordNotFound.Is(err) { - continue + conversationID := conversationID + g.Go(func() error { + conversation, err := c.GetConversation(ctx, ownerUserID, conversationID) + if err != nil { + if errs.ErrRecordNotFound.Is(err) { + return nil + } + return err } - return nil, err - } + conversationsChan <- conversation + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + close(conversationsChan) + for conversation := range conversationsChan { conversations = append(conversations, conversation) } return conversations, nil } -func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (*listMap[string], error) { - return localcache.AnyValue[*listMap[string]](c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) (any, error) { - return newListMap(c.client.GetConversationNotReceiveMessageUserIDs(ctx, conversationID)) +func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconversation.GetConversationNotReceiveMessageUserIDsResp, err error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs req", "conversationID", conversationID) + defer func() { + if err == nil { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", "conversationID", conversationID, "value", val) + } else { + log.ZError(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", err, "conversationID", conversationID) + } + }() + var cache cacheProto[pbconversation.GetConversationNotReceiveMessageUserIDsResp] + return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "conversationID", conversationID) + return cache.Marshal(c.client.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) })) } @@ -115,7 +158,7 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDs(ctx con if err != nil { return nil, err } - return res.List, nil + return res.UserIDs, nil } func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx context.Context, conversationID string) (map[string]struct{}, error) { @@ -123,5 +166,5 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx c if err != nil { return nil, err } - return res.Map, nil + return datautil.SliceSet(res.UserIDs), nil } diff --git a/pkg/rpccache/friend.go b/pkg/rpccache/friend.go index a5cee2567c..dca3b4c97c 100644 --- a/pkg/rpccache/friend.go +++ b/pkg/rpccache/friend.go @@ -16,7 +16,8 @@ package rpccache import ( "context" - cachekey2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -30,7 +31,7 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &FriendLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -46,36 +47,55 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo type FriendLocalCache struct { client rpcclient.FriendRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) { - log.ZDebug(ctx, "FriendLocalCache IsFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID) + res, err := f.isFriend(ctx, possibleFriendUserID, userID) + if err != nil { + return false, err + } + return res.InUser1Friends, nil +} + +func (f *FriendLocalCache) isFriend(ctx context.Context, possibleFriendUserID, userID string) (val *relation.IsFriendResp, err error) { + log.ZDebug(ctx, "FriendLocalCache isFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "FriendLocalCache IsFriend return", "value", val) + log.ZDebug(ctx, "FriendLocalCache isFriend return", "possibleFriendUserID", possibleFriendUserID, "userID", userID, "value", val) } else { - log.ZError(ctx, "FriendLocalCache IsFriend return", err) + log.ZError(ctx, "FriendLocalCache isFriend return", err, "possibleFriendUserID", possibleFriendUserID, "userID", userID) } }() - return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) (any, error) { - log.ZDebug(ctx, "FriendLocalCache IsFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) - return f.client.IsFriend(ctx, possibleFriendUserID, userID) - }, cachekey2.GetFriendIDsKey(possibleFriendUserID))) + var cache cacheProto[relation.IsFriendResp] + return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "FriendLocalCache isFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) + return cache.Marshal(f.client.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) + }, cachekey.GetFriendIDsKey(possibleFriendUserID))) } // IsBlack possibleBlackUserID selfUserID. func (f *FriendLocalCache) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (val bool, err error) { - log.ZDebug(ctx, "FriendLocalCache IsBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID) + res, err := f.isBlack(ctx, possibleBlackUserID, userID) + if err != nil { + return false, err + } + return res.InUser2Blacks, nil +} + +// IsBlack possibleBlackUserID selfUserID. +func (f *FriendLocalCache) isBlack(ctx context.Context, possibleBlackUserID, userID string) (val *relation.IsBlackResp, err error) { + log.ZDebug(ctx, "FriendLocalCache isBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "FriendLocalCache IsBlack return", "value", val) + log.ZDebug(ctx, "FriendLocalCache isBlack return", "possibleBlackUserID", possibleBlackUserID, "userID", userID, "value", val) } else { - log.ZError(ctx, "FriendLocalCache IsBlack return", err) + log.ZError(ctx, "FriendLocalCache isBlack return", err, "possibleBlackUserID", possibleBlackUserID, "userID", userID) } }() - return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) (any, error) { + var cache cacheProto[relation.IsBlackResp] + return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID) - return f.client.IsBlack(ctx, possibleBlackUserID, userID) - }, cachekey2.GetBlackIDsKey(userID))) + return cache.Marshal(f.client.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) + }, cachekey.GetBlackIDsKey(userID))) } diff --git a/pkg/rpccache/group.go b/pkg/rpccache/group.go index 55e1438be3..b2d852fc5f 100644 --- a/pkg/rpccache/group.go +++ b/pkg/rpccache/group.go @@ -17,6 +17,8 @@ package rpccache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -32,7 +34,7 @@ func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.Loca log.ZDebug(context.Background(), "GroupLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &GroupLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -48,21 +50,22 @@ func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.Loca type GroupLocalCache struct { client rpcclient.GroupRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } -func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *listMap[string], err error) { +func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *group.GetGroupMemberUserIDsResp, err error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs req", "groupID", groupID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs return", "value", val) + log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs return", "groupID", groupID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache getGroupMemberIDs return", err) + log.ZError(ctx, "GroupLocalCache getGroupMemberIDs return", err, "groupID", groupID) } }() - return localcache.AnyValue[*listMap[string]](g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) (any, error) { + var cache cacheProto[group.GetGroupMemberUserIDsResp] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs rpc", "groupID", groupID) - return newListMap(g.client.GetGroupMemberIDs(ctx, groupID)) + return cache.Marshal(g.client.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) })) } @@ -70,14 +73,15 @@ func (g *GroupLocalCache) GetGroupMember(ctx context.Context, groupID, userID st log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID, "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val) + log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "groupID", groupID, "userID", userID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err) + log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err, "groupID", groupID, "userID", userID) } }() - return localcache.AnyValue[*sdkws.GroupMemberFullInfo](g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.GroupMemberFullInfo] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID, "userID", userID) - return g.client.GetGroupMemberCache(ctx, groupID, userID) + return cache.Marshal(g.client.GetGroupMemberCache(ctx, groupID, userID)) })) } @@ -85,14 +89,15 @@ func (g *GroupLocalCache) GetGroupInfo(ctx context.Context, groupID string) (val log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID) defer func() { if err == nil { - log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val) + log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "groupID", groupID, "value", val) } else { - log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err) + log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err, "groupID", groupID) } }() - return localcache.AnyValue[*sdkws.GroupInfo](g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.GroupInfo] + return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID) - return g.client.GetGroupInfoCache(ctx, groupID) + return cache.Marshal(g.client.GetGroupInfoCache(ctx, groupID)) })) } @@ -101,7 +106,7 @@ func (g *GroupLocalCache) GetGroupMemberIDs(ctx context.Context, groupID string) if err != nil { return nil, err } - return res.List, nil + return res.UserIDs, nil } func (g *GroupLocalCache) GetGroupMemberIDMap(ctx context.Context, groupID string) (map[string]struct{}, error) { @@ -109,7 +114,7 @@ func (g *GroupLocalCache) GetGroupMemberIDMap(ctx context.Context, groupID strin if err != nil { return nil, err } - return res.Map, nil + return datautil.SliceSet(res.UserIDs), nil } func (g *GroupLocalCache) GetGroupInfos(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go new file mode 100644 index 0000000000..a02a0662d4 --- /dev/null +++ b/pkg/rpccache/online.go @@ -0,0 +1,330 @@ +package rpccache + +import ( + "context" + "fmt" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" + "math/rand" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/db/cacheutil" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" +) + +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { + l := &sync.Mutex{} + x := &OnlineCache{ + user: user, + group: group, + fullUserCache: fullUserCache, + Lock: l, + Cond: sync.NewCond(l), + } + + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + + switch x.fullUserCache { + case true: + log.ZDebug(ctx, "fullUserCache is true") + x.mapCache = cacheutil.NewCache[string, []int32]() + go func() { + if err := x.initUsersOnlineStatus(ctx); err != nil { + log.ZError(ctx, "initUsersOnlineStatus failed", err) + } + }() + case false: + log.ZDebug(ctx, "fullUserCache is false") + x.lruCache = lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + }) + x.CurrentPhase.Store(DoSubscribeOver) + x.Cond.Broadcast() + } + + go func() { + x.doSubscribe(ctx, rdb, fn) + }() + return x, nil +} + +const ( + Begin uint32 = iota + DoOnlineStatusOver + DoSubscribeOver +) + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + + // fullUserCache if enabled, caches the online status of all users using mapCache; + // otherwise, only a portion of users' online statuses (regardless of whether they are online) will be cached using lruCache. + fullUserCache bool + + lruCache lru.LRU[string, []int32] + mapCache *cacheutil.Cache[string, []int32] + + Lock *sync.Mutex + Cond *sync.Cond + CurrentPhase atomic.Uint32 +} + +func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { + log.ZDebug(ctx, "init users online status begin") + + var ( + totalSet atomic.Int64 + maxTries = 5 + retryInterval = time.Second * 5 + + resp *user.GetAllOnlineUsersResp + ) + + defer func(t time.Time) { + log.ZInfo(ctx, "init users online status end", "cost", time.Since(t), "totalSet", totalSet.Load()) + o.CurrentPhase.Store(DoOnlineStatusOver) + o.Cond.Broadcast() + }(time.Now()) + + retryOperation := func(operation func() error, operationName string) error { + for i := 0; i < maxTries; i++ { + if err = operation(); err != nil { + log.ZWarn(ctx, fmt.Sprintf("initUsersOnlineStatus: %s failed", operationName), err) + time.Sleep(retryInterval) + } else { + return nil + } + } + return err + } + + cursor := uint64(0) + for resp == nil || resp.NextCursor != 0 { + if err = retryOperation(func() error { + resp, err = o.user.GetAllOnlineUsers(ctx, cursor) + if err != nil { + return err + } + + for _, u := range resp.StatusList { + if u.Status == constant.Online { + o.setUserOnline(u.UserID, u.PlatformIDs) + } + totalSet.Add(1) + } + cursor = resp.NextCursor + return nil + }, "getAllOnlineUsers"); err != nil { + return err + } + } + + return nil +} + +func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) { + o.Lock.Lock() + ch := rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() + for o.CurrentPhase.Load() < DoOnlineStatusOver { + o.Cond.Wait() + } + o.Lock.Unlock() + log.ZInfo(ctx, "begin doSubscribe") + + doMessage := func(message *redis.Message) { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache setHasUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + return + } + log.ZDebug(ctx, fmt.Sprintf("get subscribe %s message", cachekey.OnlineChannel), "useID", userID, "platformIDs", platformIDs) + switch o.fullUserCache { + case true: + if len(platformIDs) == 0 { + // offline + o.mapCache.Delete(userID) + } else { + o.mapCache.Store(userID, platformIDs) + } + case false: + storageCache := o.setHasUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setHasUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + if fn != nil { + fn(ctx, userID, platformIDs) + } + } + } + + if o.CurrentPhase.Load() == DoOnlineStatusOver { + for done := false; !done; { + select { + case message := <-ch: + doMessage(message) + default: + o.CurrentPhase.Store(DoSubscribeOver) + o.Cond.Broadcast() + done = true + } + } + } + + for message := range ch { + doMessage(message) + } +} + +func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { + return o.user.GetUserOnlinePlatform(ctx, userID) + }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) + return nil, err + } + //log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + return platformIDs, nil +} + +func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + platformIDs, err := o.getUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + tmp := make([]int32, len(platformIDs)) + copy(tmp, platformIDs) + return platformIDs, nil +} + +// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { +// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) +// if err != nil { +// return nil, err +// } +// tmp := make([]int32, len(platformIDs)) +// copy(tmp, platformIDs) +// return platformIDs, nil +// } + +func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { + platformIDs, err := o.getUserOnlinePlatform(ctx, userID) + if err != nil { + return false, err + } + return len(platformIDs) > 0, nil +} + +func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { + platformIDsMap := make(map[string][]int32) + + usersStatus, err := o.user.GetUsersOnlinePlatform(ctx, missingUsers) + if err != nil { + return nil, err + } + + for _, u := range usersStatus { + platformIDsMap[u.UserID] = u.PlatformIDs + } + + return platformIDsMap, nil + }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userIDs) + return nil, err + } + return platformIDsMap, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { + t := time.Now() + + var ( + onlineUserIDs = make([]string, 0, len(userIDs)) + offlineUserIDs = make([]string, 0, len(userIDs)) + ) + + switch o.fullUserCache { + case true: + for _, userID := range userIDs { + if _, ok := o.mapCache.Load(userID); ok { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + case false: + userOnlineMap, err := o.getUserOnlinePlatformBatch(ctx, userIDs) + if err != nil { + return nil, nil, err + } + + for key, value := range userOnlineMap { + if len(value) > 0 { + onlineUserIDs = append(onlineUserIDs, key) + } else { + offlineUserIDs = append(offlineUserIDs, key) + } + } + } + + log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return userIDs, offlineUserIDs, nil +} + +//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { +// onlineUserIDs := make([]string, 0, len(userIDs)) +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) +// return onlineUserIDs, nil +//} +// +//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// var onlineUserIDs []string +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) +// return onlineUserIDs, nil +//} + +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { + switch o.fullUserCache { + case true: + o.mapCache.Store(userID, platformIDs) + case false: + o.lruCache.Set(userID, platformIDs) + } +} + +func (o *OnlineCache) setHasUserOnline(userID string, platformIDs []int32) bool { + return o.lruCache.SetHas(userID, platformIDs) +} diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 25a8eb20d4..7c676f30a1 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -16,12 +16,12 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -32,7 +32,7 @@ func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalC log.ZDebug(context.Background(), "UserLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &UserLocalCache{ client: client, - local: localcache.New[any]( + local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), localcache.WithLinkSlotNum(lc.SlotNum), @@ -48,7 +48,7 @@ func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalC type UserLocalCache struct { client rpcclient.UserRpcClient - local localcache.Cache[any] + local localcache.Cache[[]byte] } func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *sdkws.UserInfo, err error) { @@ -60,24 +60,34 @@ func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *s log.ZError(ctx, "UserLocalCache GetUserInfo return", err) } }() - return localcache.AnyValue[*sdkws.UserInfo](u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) (any, error) { + var cache cacheProto[sdkws.UserInfo] + return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserInfo rpc", "userID", userID) - return u.client.GetUserInfo(ctx, userID) + return cache.Marshal(u.client.GetUserInfo(ctx, userID)) })) } func (u *UserLocalCache) GetUserGlobalMsgRecvOpt(ctx context.Context, userID string) (val int32, err error) { - log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt req", "userID", userID) + resp, err := u.getUserGlobalMsgRecvOpt(ctx, userID) + if err != nil { + return 0, err + } + return resp.GlobalRecvMsgOpt, nil +} + +func (u *UserLocalCache) getUserGlobalMsgRecvOpt(ctx context.Context, userID string) (val *user.GetGlobalRecvMessageOptResp, err error) { + log.ZDebug(ctx, "UserLocalCache getUserGlobalMsgRecvOpt req", "userID", userID) defer func() { if err == nil { - log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", "value", val) + log.ZDebug(ctx, "UserLocalCache getUserGlobalMsgRecvOpt return", "value", val) } else { - log.ZError(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", err) + log.ZError(ctx, "UserLocalCache getUserGlobalMsgRecvOpt return", err) } }() - return localcache.AnyValue[int32](u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) (any, error) { + var cache cacheProto[user.GetGlobalRecvMessageOptResp] + return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) - return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) + return cache.Marshal(u.client.Client.GetGlobalRecvMessageOpt(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) })) } diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go index e078f432b3..ccca856194 100644 --- a/pkg/rpcclient/conversation.go +++ b/pkg/rpcclient/conversation.go @@ -77,12 +77,17 @@ func (c *ConversationRpcClient) SetConversationMaxSeq(ctx context.Context, owner return err } +func (c *ConversationRpcClient) SetConversationMinSeq(ctx context.Context, ownerUserIDs []string, conversationID string, minSeq int64) error { + _, err := c.Client.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{OwnerUserID: ownerUserIDs, ConversationID: conversationID, MinSeq: minSeq}) + return err +} + func (c *ConversationRpcClient) SetConversations(ctx context.Context, userIDs []string, conversation *pbconversation.ConversationReq) error { _, err := c.Client.SetConversations(ctx, &pbconversation.SetConversationsReq{UserIDs: userIDs, Conversation: conversation}) return err } -func (c *ConversationRpcClient) UpdateConversations(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { +func (c *ConversationRpcClient) UpdateConversation(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { _, err := c.Client.UpdateConversation(ctx, conversation) return err } @@ -146,3 +151,11 @@ func (c *ConversationRpcClient) GetConversationNotReceiveMessageUserIDs(ctx cont } return resp.UserIDs, nil } + +func (c *ConversationRpcClient) GetConversationsNeedDestructMsgs(ctx context.Context) ([]*pbconversation.Conversation, error) { + resp, err := c.Client.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) + if err != nil { + return nil, err + } + return resp.Conversations, nil +} diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go index 5543afe4f7..359ed3a8b8 100644 --- a/pkg/rpcclient/friend.go +++ b/pkg/rpcclient/friend.go @@ -17,7 +17,7 @@ package rpcclient import ( "context" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" sdkws "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/system/program" @@ -26,7 +26,7 @@ import ( type Friend struct { conn grpc.ClientConnInterface - Client friend.FriendClient + Client relation.FriendClient discov discovery.SvcDiscoveryRegistry } @@ -35,7 +35,7 @@ func NewFriend(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *F if err != nil { program.ExitWithError(err) } - client := friend.NewFriendClient(conn) + client := relation.NewFriendClient(conn) return &Friend{discov: discov, conn: conn, Client: client} } @@ -47,11 +47,11 @@ func NewFriendRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName s func (f *FriendRpcClient) GetFriendsInfo( ctx context.Context, - ownerUserID, friendUserID string, + ownerUserID, relationUserID string, ) (resp *sdkws.FriendInfo, err error) { r, err := f.Client.GetDesignatedFriends( ctx, - &friend.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{friendUserID}}, + &relation.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{relationUserID}}, ) if err != nil { return nil, err @@ -60,17 +60,17 @@ func (f *FriendRpcClient) GetFriendsInfo( return } -// possibleFriendUserID Is PossibleFriendUserId's friends. +// possibleFriendUserID Is PossibleFriendUserId's relations. func (f *FriendRpcClient) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (bool, error) { - resp, err := f.Client.IsFriend(ctx, &friend.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) + resp, err := f.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) if err != nil { return false, err } return resp.InUser1Friends, nil } -func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) { - req := friend.GetFriendIDsReq{UserID: ownerUserID} +func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (relationIDs []string, err error) { + req := relation.GetFriendIDsReq{UserID: ownerUserID} resp, err := f.Client.GetFriendIDs(ctx, &req) if err != nil { return nil, err @@ -78,8 +78,8 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) return resp.FriendIDs, nil } -func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := b.Client.IsBlack(ctx, &friend.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) +func (f *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { + r, err := f.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) if err != nil { return false, err } diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index f660c74dd7..9b26a7abda 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -17,21 +17,22 @@ package rpcclient import ( "context" "encoding/json" + "time" + + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/idutil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/timeutil" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - "time" ) func newContentTypeConf(conf *config.Notification) map[int32]config.NotificationConfig { @@ -159,6 +160,15 @@ func (m *MessageRpcClient) SendMsg(ctx context.Context, req *msg.SendMsgReq) (*m return resp, nil } +// SetUserConversationsMinSeq set min seq +func (m *MessageRpcClient) SetUserConversationsMinSeq(ctx context.Context, req *msg.SetUserConversationsMinSeqReq) (*msg.SetUserConversationsMinSeqResp, error) { + resp, err := m.Client.SetUserConversationsMinSeq(ctx, req) + if err != nil { + return nil, err + } + return resp, nil +} + // GetMaxSeq retrieves the maximum sequence number from the gRPC client. // Errors during the gRPC call are wrapped to provide additional context. func (m *MessageRpcClient) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { @@ -174,6 +184,9 @@ func (m *MessageRpcClient) GetMaxSeqs(ctx context.Context, conversationIDs []str resp, err := m.Client.GetMaxSeqs(ctx, &msg.GetMaxSeqsReq{ ConversationIDs: conversationIDs, }) + if err != nil { + return nil, err + } return resp.MaxSeqs, err } @@ -182,6 +195,9 @@ func (m *MessageRpcClient) GetHasReadSeqs(ctx context.Context, userID string, co UserID: userID, ConversationIDs: conversationIDs, }) + if err != nil { + return nil, err + } return resp.MaxSeqs, err } @@ -190,6 +206,9 @@ func (m *MessageRpcClient) GetMsgByConversationIDs(ctx context.Context, docIDs [ ConversationIDs: docIDs, MaxSeqs: seqs, }) + if err != nil { + return nil, err + } return resp.MsgDatas, err } @@ -204,6 +223,19 @@ func (m *MessageRpcClient) PullMessageBySeqList(ctx context.Context, req *sdkws. return resp, nil } +func (m *MessageRpcClient) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *msg.GetConversationsHasReadAndMaxSeqReq) (*msg.GetConversationsHasReadAndMaxSeqResp, error) { + resp, err := m.Client.GetConversationsHasReadAndMaxSeq(ctx, req) + if err != nil { + // Wrap the error to provide more context if the gRPC call fails. + return nil, err + } + return resp, nil +} + +func (m *MessageRpcClient) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + return m.Client.GetSeqMessage(ctx, req) +} + func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversationID string) (int64, error) { resp, err := m.Client.GetConversationMaxSeq(ctx, &msg.GetConversationMaxSeqReq{ConversationID: conversationID}) if err != nil { @@ -252,8 +284,8 @@ func WithUserRpcClient(userRpcClient *UserRpcClient) NotificationSenderOptions { } const ( - notificationWorkerCount = 2 - notificationBufferSize = 200 + notificationWorkerCount = 16 + notificationBufferSize = 1024 * 1024 * 2 ) func NewNotificationSender(conf *config.Notification, opts ...NotificationSenderOptions) *NotificationSender { @@ -280,7 +312,8 @@ func WithRpcGetUserName() NotificationOptions { } func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) { - ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) + //ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) + ctx = context.WithoutCancel(ctx) ctx, cancel := context.WithTimeout(ctx, time.Second*time.Duration(5)) defer cancel() n := sdkws.NotificationElem{Detail: jsonutil.StructToJsonString(m)} @@ -324,6 +357,10 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co options := config.GetOptionsByNotification(optionsConfig) s.SetOptionsByContentType(ctx, options, contentType) msg.Options = options + // fill Notification OfflinePush by config + offlineInfo.Title = optionsConfig.OfflinePush.Title + offlineInfo.Desc = optionsConfig.OfflinePush.Desc + offlineInfo.Ex = optionsConfig.OfflinePush.Ext msg.OfflinePushInfo = &offlineInfo req.MsgData = &msg _, err = s.sendMsg(ctx, &req) @@ -333,7 +370,9 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co } func (s *NotificationSender) NotificationWithSessionType(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) { - s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) }) + if err := s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) }); err != nil { + log.ZWarn(ctx, "Push to queue failed", err, "sendID", sendID, "recvID", recvID, "msg", jsonutil.StructToJsonString(m)) + } } func (s *NotificationSender) Notification(ctx context.Context, sendID, recvID string, contentType int32, m proto.Message, opts ...NotificationOptions) { diff --git a/pkg/rpcclient/third.go b/pkg/rpcclient/third.go index 4c71dff6ad..7cdc60d52f 100644 --- a/pkg/rpcclient/third.go +++ b/pkg/rpcclient/third.go @@ -41,3 +41,7 @@ func NewThird(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, grafanaUrl } return &Third{discov: discov, Client: client, conn: conn, GrafanaUrl: grafanaUrl} } +func (t *Third) DeleteOutdatedData(ctx context.Context, expires int64) error { + _, err := t.Client.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expires}) + return err +} diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index aab96603eb..375cc993cb 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -169,6 +169,15 @@ func (u *UserRpcClient) Access(ctx context.Context, ownerUserID string) error { return authverify.CheckAccessV3(ctx, ownerUserID, u.imAdminUserID) } +// GetAllUserID retrieves all user IDs with pagination options. +func (u *UserRpcClient) GetAllUserID(ctx context.Context, pageNumber, showNumber int32) (*user.GetAllUserIDResp, error) { + resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) + if err != nil { + return nil, err + } + return resp, nil +} + // GetAllUserIDs retrieves all user IDs with pagination options. func (u *UserRpcClient) GetAllUserIDs(ctx context.Context, pageNumber, showNumber int32) ([]string, error) { resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) @@ -193,3 +202,29 @@ func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) }) return err } + +func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { + if len(userIDs) == 0 { + return nil, nil + } + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs, UserID: u.imAdminUserID[0]}) + if err != nil { + return nil, err + } + return resp.StatusList, nil +} + +func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + resp, err := u.GetUsersOnlinePlatform(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(resp) == 0 { + return nil, nil + } + return resp[0].PlatformIDs, nil +} + +func (u *UserRpcClient) GetAllOnlineUsers(ctx context.Context, cursor uint64) (*user.GetAllOnlineUsersResp, error) { + return u.Client.GetAllOnlineUsers(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) +} diff --git a/pkg/util/conversationutil/conversationutil.go b/pkg/util/conversationutil/conversationutil.go index 5683d8df89..f0a44ab1e1 100644 --- a/pkg/util/conversationutil/conversationutil.go +++ b/pkg/util/conversationutil/conversationutil.go @@ -19,6 +19,14 @@ func GenGroupConversationID(groupID string) string { return "sg_" + groupID } +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "sg_") +} + +func IsNotificationConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "n_") +} + func GenConversationUniqueKeyForSingle(sendID, recvID string) string { l := []string{sendID, recvID} sort.Strings(l) diff --git a/pkg/util/hashutil/id.go b/pkg/util/hashutil/id.go new file mode 100644 index 0000000000..52e7f4c6f2 --- /dev/null +++ b/pkg/util/hashutil/id.go @@ -0,0 +1,16 @@ +package hashutil + +import ( + "crypto/md5" + "encoding/binary" + "encoding/json" +) + +func IdHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} diff --git a/pkg/util/useronline/split.go b/pkg/util/useronline/split.go new file mode 100644 index 0000000000..c39d31d15b --- /dev/null +++ b/pkg/util/useronline/split.go @@ -0,0 +1,27 @@ +package useronline + +import ( + "errors" + "strconv" + "strings" +) + +func ParseUserOnlineStatus(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil +} diff --git a/scripts/create-topic.sh b/scripts/create-topic.sh index 206075fb83..bbc739287f 100755 --- a/scripts/create-topic.sh +++ b/scripts/create-topic.sh @@ -35,7 +35,7 @@ done echo "Kafka is ready. Creating topics..." -topics=("toRedis" "toMongo" "toPush") +topics=("toRedis" "toMongo" "toPush" "toOfflinePush") partitions=8 replicationFactor=1 diff --git a/start-config.yml b/start-config.yml index a9c412b33f..1231b5d0d4 100644 --- a/start-config.yml +++ b/start-config.yml @@ -3,8 +3,8 @@ serviceBinaries: openim-crontask: 1 openim-rpc-user: 1 openim-msggateway: 1 - openim-push: 1 - openim-msgtransfer: 4 + openim-push: 8 + openim-msgtransfer: 8 openim-rpc-conversation: 1 openim-rpc-auth: 1 openim-rpc-group: 1 @@ -14,4 +14,5 @@ serviceBinaries: toolBinaries: - check-free-memory - check-component + - seq maxFileDescriptors: 10000 diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5fa84ac36f..4f4c08c16a 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -18,6 +18,12 @@ import ( "context" "flag" "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/db/mongoutil" @@ -27,11 +33,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" - "io/ioutil" - "log" - "os" - "path/filepath" - "time" ) const maxRetry = 180 @@ -65,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error { } func CheckKafka(ctx context.Context, conf *config.Kafka) error { - return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic}) + return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic}) } func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) { diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go new file mode 100644 index 0000000000..2bec5a8f1c --- /dev/null +++ b/tools/seq/internal/main.go @@ -0,0 +1,331 @@ +package internal + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "gopkg.in/yaml.v3" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const ( + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" +) + +const ( + batchSize = 100 + dataVersionCollection = "data_version" + seqKey = "seq" + seqVersion = 38 +) + +func readConfig[T any](dir string, name string) (*T, error) { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return nil, err + } + var conf T + if err := yaml.Unmarshal(data, &conf); err != nil { + return nil, err + } + return &conf, nil +} + +func Main(conf string, del time.Duration) error { + redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) + if err != nil { + return err + } + mongodbConfig, err := readConfig[config.Mongo](conf, cmd.MongodbConfigFileName) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, redisConfig.Build()) + if err != nil { + return err + } + mgocli, err := mongoutil.NewMongoDB(ctx, mongodbConfig.Build()) + if err != nil { + return err + } + versionColl := mgocli.GetDB().Collection(dataVersionCollection) + converted, err := CheckVersion(versionColl, seqKey, seqVersion) + if err != nil { + return err + } + if converted { + fmt.Println("[seq] seq data has been converted") + return nil + } + if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { + return err + } + cSeq, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSeq, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSpitHasReadSeq := func(id string) (conversationID string, userID string, err error) { + // HasReadSeq + userID + ":" + conversationID + arr := strings.Split(id, ":") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) + } + userID = arr[0] + conversationID = arr[1] + return + } + uSpitConversationUserMinSeq := func(id string) (conversationID string, userID string, err error) { + // ConversationUserMinSeq + conversationID + "u:" + userID + arr := strings.Split(id, "u:") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) + } + conversationID = arr[0] + userID = arr[1] + return + } + + ts := []*taskSeq{ + { + Prefix: MaxSeq, + GetSeq: cSeq.GetMaxSeq, + SetSeq: cSeq.SetMaxSeq, + }, + { + Prefix: MinSeq, + GetSeq: cSeq.GetMinSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: HasReadSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetUserReadSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return err + } + return uSeq.SetUserReadSeq(ctx, conversationID, userID, seq) + }, + }, + { + Prefix: ConversationUserMinSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetUserMinSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return err + } + return uSeq.SetUserMinSeq(ctx, conversationID, userID, seq) + }, + }, + } + + cancel() + ctx = context.Background() + + var wg sync.WaitGroup + wg.Add(len(ts)) + + for i := range ts { + go func(task *taskSeq) { + defer wg.Done() + err := seqRedisToMongo(ctx, rdb, task.GetSeq, task.SetSeq, task.Prefix, del, &task.Count) + task.End = time.Now() + task.Error = err + }(ts[i]) + } + start := time.Now() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var buf bytes.Buffer + + printTaskInfo := func(now time.Time) { + buf.Reset() + buf.WriteString(now.Format(time.DateTime)) + buf.WriteString(" \n") + for i := range ts { + task := ts[i] + if task.Error == nil { + if task.End.IsZero() { + buf.WriteString(fmt.Sprintf("[%s] converting %s* count %d", now.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } else { + buf.WriteString(fmt.Sprintf("[%s] success %s* count %d", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } + } else { + buf.WriteString(fmt.Sprintf("[%s] failed %s* count %d error %s", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count), task.Error)) + } + buf.WriteString("\n") + } + fmt.Println(buf.String()) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case s := <-sigs: + return fmt.Errorf("exit by signal %s", s) + case <-done: + errs := make([]error, 0, len(ts)) + for i := range ts { + task := ts[i] + if task.Error != nil { + errs = append(errs, fmt.Errorf("seq %s failed %w", task.Prefix, task.Error)) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + printTaskInfo(time.Now()) + if err := SetVersion(versionColl, seqKey, seqVersion); err != nil { + return fmt.Errorf("set mongodb seq version %w", err) + } + return nil + case now := <-ticker.C: + printTaskInfo(now) + } + } +} + +type taskSeq struct { + Prefix string + Count int64 + Error error + End time.Time + GetSeq func(ctx context.Context, id string) (int64, error) + SetSeq func(ctx context.Context, id string, seq int64) error +} + +func seqRedisToMongo(ctx context.Context, rdb redis.UniversalClient, getSeq func(ctx context.Context, id string) (int64, error), setSeq func(ctx context.Context, id string, seq int64) error, prefix string, delAfter time.Duration, count *int64) error { + var ( + cursor uint64 + keys []string + err error + ) + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { + return err + } + if len(keys) > 0 { + for _, key := range keys { + seqStr, err := rdb.Get(ctx, key).Result() + if err != nil { + return fmt.Errorf("redis get %s failed %w", key, err) + } + seq, err := strconv.Atoi(seqStr) + if err != nil { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + if seq < 0 { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + id := strings.TrimPrefix(key, prefix) + redisSeq := int64(seq) + mongoSeq, err := getSeq(ctx, id) + if err != nil { + return fmt.Errorf("get mongo seq %s failed %w", key, err) + } + if mongoSeq < redisSeq { + if err := setSeq(ctx, id, redisSeq); err != nil { + return fmt.Errorf("set mongo seq %s failed %w", key, err) + } + } + if delAfter > 0 { + if err := rdb.Expire(ctx, key, delAfter).Err(); err != nil { + return fmt.Errorf("redis expire key %s failed %w", key, err) + } + } else { + if err := rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("redis del key %s failed %w", key, err) + } + } + atomic.AddInt64(count, 1) + } + } + if cursor == 0 { + return nil + } + } +} + +func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { + type VersionTable struct { + Key string `bson:"key"` + Value string `bson:"value"` + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + res, err := mongoutil.FindOne[VersionTable](ctx, coll, bson.M{"key": key}) + if err == nil { + ver, err := strconv.Atoi(res.Value) + if err != nil { + return false, fmt.Errorf("version %s parse error %w", res.Value, err) + } + if ver >= currentVersion { + return true, nil + } + return false, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return false, nil + } else { + return false, err + } +} + +func SetVersion(coll *mongo.Collection, key string, version int) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + option := options.Update().SetUpsert(true) + filter := bson.M{"key": key, "value": strconv.Itoa(version)} + update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}} + return mongoutil.UpdateOne(ctx, coll, filter, update, false, option) +} diff --git a/tools/seq/main.go b/tools/seq/main.go new file mode 100644 index 0000000000..16da9f156e --- /dev/null +++ b/tools/seq/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "time" +) + +func main() { + var ( + config string + second int + ) + flag.StringVar(&config, "c", "", "config directory") + flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") + flag.Parse() + if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { + fmt.Println("seq task", err) + } + fmt.Println("seq task success!") +} diff --git a/tools/url2im/pkg/manage.go b/tools/url2im/pkg/manage.go index 5e1626da9f..3664baa25b 100644 --- a/tools/url2im/pkg/manage.go +++ b/tools/url2im/pkg/manage.go @@ -234,7 +234,7 @@ func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) { } for i, currentPartSize := range part.PartSizes { md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize)) - if m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil { + if err := m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil { return "", err } if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] { diff --git a/version/version b/version/version new file mode 100644 index 0000000000..0be1fc7d24 --- /dev/null +++ b/version/version @@ -0,0 +1 @@ +3.8.0 \ No newline at end of file diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000000..23b3a82f53 --- /dev/null +++ b/version/version.go @@ -0,0 +1,6 @@ +package version + +import _ "embed" + +//go:embed version +var Version string