From c9df03fd353cabf43ad9fbe8d70be663e3729d3a Mon Sep 17 00:00:00 2001 From: HUAHUAI23 Date: Fri, 28 Jun 2024 07:54:05 +0000 Subject: [PATCH 1/2] ok --- .../dedicated-database-task.service.ts | 7 ++++++- .../dedicated-database.service.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/database/dedicated-database/dedicated-database-task.service.ts b/server/src/database/dedicated-database/dedicated-database-task.service.ts index fbe42d7..8d4b87c 100644 --- a/server/src/database/dedicated-database/dedicated-database-task.service.ts +++ b/server/src/database/dedicated-database/dedicated-database-task.service.ts @@ -6,7 +6,7 @@ import { DedicatedDatabaseState, } from '../entities/dedicated-database' import { DedicatedDatabaseService } from './dedicated-database.service' -import { TASK_LOCK_INIT_TIME } from 'src/constants' +import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants' import { Injectable, Logger } from '@nestjs/common' import { RegionService } from 'src/region/region.service' import { ClusterService } from 'src/region/cluster/cluster.service' @@ -25,6 +25,10 @@ export class DedicatedDatabaseTaskService { @Cron(CronExpression.EVERY_SECOND) async tick() { + if (ServerConfig.DISABLED_INSTANCE_TASK) { + return + } + this.handleDeletingPhase().catch((err) => { this.logger.error(err) }) @@ -95,6 +99,7 @@ export class DedicatedDatabaseTaskService { return } + await this.dbService.databaseConnectionIsOk(appid) manifest = await this.dbService.getDeployManifest(region, user, appid) const unavailable = manifest?.status?.phase !== 'Running' if (unavailable) { diff --git a/server/src/database/dedicated-database/dedicated-database.service.ts b/server/src/database/dedicated-database/dedicated-database.service.ts index c9840f9..aa20c5a 100644 --- a/server/src/database/dedicated-database/dedicated-database.service.ts +++ b/server/src/database/dedicated-database/dedicated-database.service.ts @@ -387,6 +387,19 @@ export class DedicatedDatabaseService { } } + async databaseConnectionIsOk(appid: string) { + try { + const { client } = await this.findAndConnect(appid) + const admin = client.db('admin') + const replSetStatus = await admin.command({ replSetGetStatus: 1 }) + console.log(replSetStatus) + + return true + } catch (error) { + return false + } + } + async checkDatabaseSyncLimit(uid: ObjectId) { const count = await SystemDatabase.db .collection('DatabaseSyncRecord') From d403a29f665a67c872866743e86f96e4daccc8db Mon Sep 17 00:00:00 2001 From: HUAHUAI23 Date: Fri, 28 Jun 2024 09:47:18 +0000 Subject: [PATCH 2/2] ok --- .../dedicated-database-task.service.ts | 6 ++-- .../dedicated-database.service.ts | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/src/database/dedicated-database/dedicated-database-task.service.ts b/server/src/database/dedicated-database/dedicated-database-task.service.ts index 8d4b87c..83f9d49 100644 --- a/server/src/database/dedicated-database/dedicated-database-task.service.ts +++ b/server/src/database/dedicated-database/dedicated-database-task.service.ts @@ -99,10 +99,12 @@ export class DedicatedDatabaseTaskService { return } - await this.dbService.databaseConnectionIsOk(appid) + const connectionOk = await this.dbService.databaseConnectionIsOk(appid) + manifest = await this.dbService.getDeployManifest(region, user, appid) const unavailable = manifest?.status?.phase !== 'Running' - if (unavailable) { + + if (unavailable && !connectionOk) { await this.relock(appid, waitingTime) return } diff --git a/server/src/database/dedicated-database/dedicated-database.service.ts b/server/src/database/dedicated-database/dedicated-database.service.ts index aa20c5a..4af52b5 100644 --- a/server/src/database/dedicated-database/dedicated-database.service.ts +++ b/server/src/database/dedicated-database/dedicated-database.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common' +import { Injectable, Logger } from '@nestjs/common' import { Region } from 'src/region/entities/region' import { DedicatedDatabase, @@ -36,6 +36,8 @@ const p_exec = promisify(exec) @Injectable() export class DedicatedDatabaseService { + private readonly logger = new Logger(DedicatedDatabase.name) + constructor( private readonly cluster: ClusterService, private readonly mongoService: MongoService, @@ -288,7 +290,7 @@ export class DedicatedDatabaseService { `mongodump --uri='${connectionUri}' --gzip --archive=${filePath}`, ) } catch (error) { - console.error(`failed to export db ${appid}`, error) + this.logger.error(`failed to export db ${appid}`, error) throw error } finally { await SystemDatabase.db @@ -387,15 +389,36 @@ export class DedicatedDatabaseService { } } - async databaseConnectionIsOk(appid: string) { + async databaseConnectionIsOk(appid: string): Promise { try { const { client } = await this.findAndConnect(appid) const admin = client.db('admin') const replSetStatus = await admin.command({ replSetGetStatus: 1 }) - console.log(replSetStatus) + const members = replSetStatus.members + const replicaSetOk: boolean = replSetStatus.ok === 1 + + const healthyMembers = members.filter( + (member: any) => member.health === 1, + ) + + const primary = healthyMembers.find( + (member: any) => member.stateStr === 'PRIMARY', + ) - return true + const majorityCount = Math.ceil(members.length / 2) + + const isClusterHealthy = + replicaSetOk && primary && healthyMembers.length >= majorityCount + + if (isClusterHealthy) { + return true + } + + return false } catch (error) { + this.logger.verbose( + `dedicatedDatabase health check failed ${appid}\n${error.message}`, + ) return false } }