3.6.1 code conventions

.dockerignore

@@ -5,9 +5,8 @@
 .git
 
 # Ignore build artifacts
-_output/
 logs/
-
+_output/
 # Ignore non-essential documentation
 README.md
 README-zh_CN.md
@@ -18,8 +17,6 @@ CHANGELOG/
 # Ignore testing and linting configuration
 .golangci.yml
 
-# Ignore deployment-related files
-docker-compose.yaml
 
 # Ignore assets
 assets/

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

Dockerfile

@@ -1,30 +1,49 @@
-# Build Stage
-FROM golang:1.20 AS builder
+# Use Go 1.21 Alpine as the base image for building the application
+FROM golang:1.21-alpine as builder
 
-# Set go mod installation source and proxy
-ARG GO111MODULE=on
+# Define the base directory for the application as an environment variable
+ENV SERVER_DIR=/openim-server
 
-ENV GO111MODULE=$GO111MODULE
-ENV GOPROXY=$GOPROXY
+# Set the working directory inside the container based on the environment variable
+WORKDIR $SERVER_DIR
 
-# Set up the working directory
-WORKDIR /openim/openim-server
+# Set the Go proxy to improve dependency resolution speed
+ENV GOPROXY=https://goproxy.io,direct
 
+# Copy all files from the current directory into the container
+COPY . .
 
-# Copy all files to the container
-ADD . .
+RUN go mod download
 
-RUN make clean
-RUN make build
+# Install Mage to use for building the application
+RUN go install github.com/magefile/[email protected]
 
-FROM ghcr.io/openim-sigs/openim-ubuntu-image:latest
+# Optionally build your application if needed
+RUN mage build
 
-WORKDIR ${SERVER_WORKDIR}
+# Using Alpine Linux with Go environment for the final image
+FROM golang:1.21-alpine
 
-# Copy scripts and binary files to the production image
-COPY --from=builder ${OPENIM_SERVER_BINDIR} /openim/openim-server/_output/bin
-COPY --from=builder ${OPENIM_SERVER_CMDDIR} /openim/openim-server/scripts
-COPY --from=builder ${SERVER_WORKDIR}/config /openim/openim-server/config
-COPY --from=builder ${SERVER_WORKDIR}/deployments /openim/openim-server/deployments
+# Install necessary packages, such as bash
+RUN apk add --no-cache bash
 
-CMD ["/openim/openim-server/scripts/docker-start-all.sh"]
+# Set the environment and work directory
+ENV SERVER_DIR=/openim-server
+WORKDIR $SERVER_DIR
+
+
+# Copy the compiled binaries and mage from the builder image to the final image
+COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output
+COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config
+COPY --from=builder /go/bin/mage /usr/local/bin/mage
+COPY --from=builder $SERVER_DIR/magefile_windows.go $SERVER_DIR/
+COPY --from=builder $SERVER_DIR/magefile_unix.go $SERVER_DIR/
+COPY --from=builder $SERVER_DIR/magefile.go $SERVER_DIR/
+COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/
+COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/
+COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/
+
+RUN go get github.com/openimsdk/[email protected]
+
+# Set the command to run when the container starts
+ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"]

config/README.md

@@ -0,0 +1,63 @@
+---
+title: 'OpenIM Configuration Files and Common Configuration Item Modifications Guide'
+
+## Configuration Files Explanation
+
+| Configuration File              | Description                                                  |
+| ------------------------------- | ------------------------------------------------------------ |
+| **kafka.yml**                   | Configurations for Kafka username, password, address, etc.   |
+| **redis.yml**                   | Configurations for Redis password, address, etc.             |
+| **minio.yml**                   | Configurations for MinIO username, password, address, and external IP/domain; failing to modify external IP or domain may cause image file sending failures |
+| **zookeeper.yml**               | Configurations for ZooKeeper user, password, address, etc.   |
+| **mongodb.yml**                 | Configurations for MongoDB username, password, address, etc. |
+| **log.yml**                     | Configurations for log level and storage directory.          |
+| **notification.yml**            | Configurations for events like adding friends, creating groups, etc. |
+| **share.yml**                   | Common configurations needed by various OpenIM services, such as secret. |
+| **webhooks.yml**                | Configurations for URLs in Webhook.                          |
+| **local-cache.yml**             | Local cache configurations.                                  |
+| **openim-rpc-third.yml**        | Configurations for listening IP, port, and storage settings for images and videos in openim-rpc-third service. |
+| **openim-rpc-user.yml**         | Configurations for listening IP and port in openim-rpc-user service. |
+| **openim-api.yml**              | Configurations for listening IP, port, etc., in openim-api service. |
+| **openim-crontask.yml**         | Configurations for openim-crontask service.                  |
+| **openim-msggateway.yml**       | Configurations for listening IP, port, etc., in openim-msggateway service. |
+| **openim-msgtransfer.yml**      | Configurations for openim-msgtransfer service.               |
+| **openim-push.yml**             | Configurations for listening IP, port, and offline push settings in openim-push service. |
+| **openim-rpc-auth.yml**         | Configurations for listening IP, port, and token expiration settings in openim-rpc-auth service. |
+| **openim-rpc-conversation.yml** | Configurations for listening IP, port, etc., in openim-rpc-conversation service. |
+| **openim-rpc-friend.yml**       | Configurations for listening IP, port, etc., in openim-rpc-friend service. |
+| **openim-rpc-group.yml**        | Configurations for listening IP, port, etc., in openim-rpc-group service. |
+| **openim-rpc-msg.yml**          | Configurations for listening IP, port, and whether to verify friendship before sending messages in openim-rpc-msg service. |
+
+## Common Configuration Item Modifications
+
+| Configuration Item Modification                       | Configuration File      |
+| ----------------------------------------------------- | ----------------------- |
+| Using MinIO for image and video file object storage   | `minio.yml`             |
+| Adjusting production environment logs                 | `log.yml`               |
+| Verifying friendship before sending messages          | `openim-rpc-msg.yml`    |
+| Modifying secret                                      | `share.yml`             |
+| Using OSS, COS, AWS, Kodo for image and video storage | `openim-rpc-third.yml`  |
+| Setting multiple login policy                         | `openim-msggateway.yml` |
+| Setting up offline push                               | `openim-push.yml`       |
+
+## Starting Multiple Instances of an OpenIM Service
+
+To start multiple instances of an OpenIM service, simply increase the corresponding port numbers and modify the `start-config.yml` file in the project root directory. Restart the service to take effect. For example, the configuration to start 2 instances of `openim-rpc-user` is as follows:
+
+```yaml
+rpc:
+  registerIP: ''
+  listenIP: 0.0.0.0
+  ports: [ 10110, 10111 ]
+
+prometheus:
+  enable: true
+  ports: [ 20100, 20101 ]
+```
+
+Modify `start-config.yml`:
+
+```yaml
+serviceBinaries:
+  openim-rpc-user: 2
+```

config/minio.yml

@@ -1,9 +1,7 @@
 bucket: "openim"
-port: 10005
 accessKeyID: "root"
 secretAccessKey: "openIM123"
 sessionToken: ''
-internalIP: localhost
-externalIP: 150.109.93.151
-url: https://image.rentsoft.cn/
-publicRead: false
+internalAddress: "minio:9000"
+externalAddress: "http://external_ip:10005"
+publicRead: false

go.mod

@@ -13,9 +13,8 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/openimsdk/localcache v0.0.1
 	github.com/openimsdk/protocol v0.0.64
-	github.com/openimsdk/tools v0.0.47-alpha.42
+	github.com/openimsdk/tools v0.0.49-alpha.2
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.18.0
 	github.com/stretchr/testify v1.9.0
@@ -32,9 +31,10 @@ require (
 	github.com/IBM/sarama v1.43.0
 	github.com/fatih/color v1.14.1
 	github.com/go-redis/redis v6.15.9+incompatible
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/kelindar/bitmap v1.5.2
 	github.com/likexian/gokit v0.25.13
-	github.com/openimsdk/gomake v0.0.6
+	github.com/openimsdk/gomake v0.0.9
 	github.com/redis/go-redis/v9 v9.4.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/shirou/gopsutil v3.21.11+incompatible
@@ -83,7 +83,6 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -173,5 +172,3 @@ require (
 	golang.org/x/crypto v0.21.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
-
-replace github.com/openimsdk/localcache => ./pkg/localcache

go.sum

@@ -266,12 +266,12 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
-github.com/openimsdk/gomake v0.0.6 h1:bJmQWDHBj8PQ7oGJ2SL3Gsx0k5CdI/BPfGzlGcV105s=
-github.com/openimsdk/gomake v0.0.6/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
+github.com/openimsdk/gomake v0.0.9 h1:ouf25ygN2PMQ68Gfgns/EQRPiLPnp+77SIr68GfE+n4=
+github.com/openimsdk/gomake v0.0.9/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
 github.com/openimsdk/protocol v0.0.64 h1:OrjSs4CgKN9VLvJvrAsc37O7Ru0E0VllXZQSmG/ab7U=
 github.com/openimsdk/protocol v0.0.64/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
-github.com/openimsdk/tools v0.0.47-alpha.42 h1:wM6t9otTLhXECq8aQcYaZGvBgo/ZAmbNTqVt3g3NHGg=
-github.com/openimsdk/tools v0.0.47-alpha.42/go.mod h1:P4oGP1Pd+d4ctbLD5U/XQTgl8yu8Hd3skx640Fr69ko=
+github.com/openimsdk/tools v0.0.49-alpha.2 h1:8IfV6o2ySU7C54sh/MG7ctEp1h3lSNe03OCUDWSk5Ws=
+github.com/openimsdk/tools v0.0.49-alpha.2/go.mod h1:P4oGP1Pd+d4ctbLD5U/XQTgl8yu8Hd3skx640Fr69ko=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=

internal/api/init.go

@@ -64,10 +64,6 @@ func Start(ctx context.Context, index int, config *Config) error {
 		return errs.WrapMsg(err, "failed to register discovery service")
 	}
 
-	if err = client.CreateRpcRootNodes(config.Share.RpcRegisterName.GetServiceNames()); err != nil {
-		return errs.WrapMsg(err, "failed to create RPC root nodes")
-	}
-
 	var (
 		netDone = make(chan struct{}, 1)
 		netErr  error

internal/msgtransfer/init.go

@@ -80,11 +80,6 @@ func Start(ctx context.Context, index int, config *Config) error {
 	if err != nil {
 		return err
 	}
-
-	if err := client.CreateRpcRootNodes(config.Share.RpcRegisterName.GetServiceNames()); err != nil {
-		return err
-	}
-
 	client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin")))
 	//todo MsgCacheTimeout

internal/msgtransfer/online_history_msg_handler.go

@@ -394,7 +394,7 @@ func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(
 			break
 		}
 	}
-	log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset",
+	log.ZInfo(context.Background(), "online new session msg come", "highWaterMarkOffset",
 		claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition())
 
 	var (

internal/push/push_handler.go

@@ -1,4 +1,4 @@
 // Copyright © 2023 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -96,7 +96,7 @@
 	var err error
 	switch msgFromMQ.MsgData.SessionType {
 	case constant.ReadGroupChatType:
-		err = c.Push2SuperGroup(ctx, pbData.MsgData.GroupID, pbData.MsgData)
+		err = c.Push2Group(ctx, pbData.MsgData.GroupID, pbData.MsgData)
 	default:
 		var pushUserIDList []string
 		isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync)
@@ -108,7 +108,7 @@
 		err = c.Push2User(ctx, pushUserIDList, pbData.MsgData)
 	}
 	if err != nil {
-		log.ZError(ctx, "push failed", err, "msg", pbData.String())
+		log.ZWarn(ctx, "push failed", err, "msg", pbData.String())
 	}
 }
 
@@ -179,7 +179,7 @@
 	return true
 }
 
-func (c *ConsumerHandler) Push2SuperGroup(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) {
+func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) {
 	log.ZDebug(ctx, "Get super group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID)
 	var pushToUserIDs []string
 	if err = c.webhookBeforeGroupOnlinePush(ctx, &c.config.WebhooksConfig.BeforeGroupOnlinePush, groupID, msg,

internal/rpc/third/third.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
-	"net/url"
 	"time"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
@@ -29,7 +28,6 @@ import (
 	"github.com/openimsdk/tools/db/mongoutil"
 	"github.com/openimsdk/tools/db/redisutil"
 	"github.com/openimsdk/tools/discovery"
-	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/s3"
 	"github.com/openimsdk/tools/s3/cos"
 	"github.com/openimsdk/tools/s3/minio"
@@ -38,7 +36,6 @@ import (
 )
 
 type thirdServer struct {
-	apiURL        string
 	thirdDatabase controller.ThirdDatabase
 	s3dataBase    controller.S3Database
 	userRpcClient rpcclient.UserRpcClient
@@ -73,18 +70,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg
 	if err != nil {
 		return err
 	}
-	apiURL := config.MinioConfig.URL
-	if apiURL == "" {
-		return errs.Wrap(fmt.Errorf("api is empty"))
-	}
-	if _, err := url.Parse(config.MinioConfig.URL); err != nil {
-		return err
-	}
-	if apiURL[len(apiURL)-1] != '/' {
-		apiURL += "/"
-	}
-	apiURL += "object/"
-
 	// Select the oss method according to the profile policy
 	enable := config.RpcConfig.Object.Enable
 	var o s3.Interface
@@ -103,7 +88,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg
 	}
 	cache.InitLocalCache(&config.LocalCacheConfig)
 	third.RegisterThirdServer(server, &thirdServer{
-		apiURL:        apiURL,
 		thirdDatabase: controller.NewThirdDatabase(cache.NewThirdCache(rdb), logdb),
 		userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID),
 		s3dataBase:    controller.NewS3Database(rdb, o, s3db),

pkg/common/config/config.go

@@ -22,6 +22,7 @@ import (
 	"github.com/openimsdk/tools/s3/cos"
 	"github.com/openimsdk/tools/s3/minio"
 	"github.com/openimsdk/tools/s3/oss"
+	"net"
 	"time"
 )
 
@@ -52,13 +53,11 @@ type Log struct {
 
 type Minio struct {
 	Bucket          string `mapstructure:"bucket"`
-	Port            int    `mapstructure:"port"`
 	AccessKeyID     string `mapstructure:"accessKeyID"`
 	SecretAccessKey string `mapstructure:"secretAccessKey"`
 	SessionToken    string `mapstructure:"sessionToken"`
-	InternalIP      string `mapstructure:"internalIP"`
-	ExternalIP      string `mapstructure:"externalIP"`
-	URL             string `mapstructure:"url"`
+	InternalAddress string `mapstructure:"internalAddress"`
+	ExternalAddress string `mapstructure:"externalAddress"`
 	PublicRead      bool   `mapstructure:"publicRead"`
 }
 
@@ -477,16 +476,24 @@ func (k *Kafka) Build() *kafka.Config {
 	}
 }
 func (m *Minio) Build() *minio.Config {
-	return &minio.Config{
+	conf := minio.Config{
 		Bucket:          m.Bucket,
-		Endpoint:        fmt.Sprintf("http://%s:%d", m.InternalIP, m.Port),
 		AccessKeyID:     m.AccessKeyID,
 		SecretAccessKey: m.SecretAccessKey,
 		SessionToken:    m.SessionToken,
-		SignEndpoint:    fmt.Sprintf("http://%s:%d", m.ExternalIP, m.Port),
 		PublicRead:      m.PublicRead,
 	}
-
+	if _, _, err := net.SplitHostPort(m.InternalAddress); err == nil {
+		conf.Endpoint = fmt.Sprintf("http://%s", m.InternalAddress)
+	} else {
+		conf.Endpoint = m.InternalAddress
+	}
+	if _, _, err := net.SplitHostPort(m.ExternalAddress); err == nil {
+		conf.SignEndpoint = fmt.Sprintf("http://%s", m.ExternalAddress)
+	} else {
+		conf.SignEndpoint = m.ExternalAddress
+	}
+	return &conf
 }
 func (c *Cos) Build() *cos.Config {
 	return &cos.Config{

pkg/common/config/load_config_test.go

@@ -31,6 +31,8 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) {
 	var user User
 	err := LoadConfig("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", &user)
 	assert.Nil(t, err)
+	//export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0"
 	assert.Equal(t, "0.0.0.0", user.RPC.ListenIP)
-	assert.Equal(t, []int{10110}, user.RPC.Ports)
+	//export IMENV_OPENIM_RPC_USER_RPC_PORTS="10110,10111,10112"
+	assert.Equal(t, []int{10110, 10111, 10112}, user.RPC.Ports)
 }