diff --git a/server/src/database/dedicated-database/dedicated-database-task.service.ts b/server/src/database/dedicated-database/dedicated-database-task.service.ts index fbe42d7..83f9d49 100644 --- a/server/src/database/dedicated-database/dedicated-database-task.service.ts +++ b/server/src/database/dedicated-database/dedicated-database-task.service.ts @@ -6,7 +6,7 @@ import { DedicatedDatabaseState, } from '../entities/dedicated-database' import { DedicatedDatabaseService } from './dedicated-database.service' -import { TASK_LOCK_INIT_TIME } from 'src/constants' +import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants' import { Injectable, Logger } from '@nestjs/common' import { RegionService } from 'src/region/region.service' import { ClusterService } from 'src/region/cluster/cluster.service' @@ -25,6 +25,10 @@ export class DedicatedDatabaseTaskService { @Cron(CronExpression.EVERY_SECOND) async tick() { + if (ServerConfig.DISABLED_INSTANCE_TASK) { + return + } + this.handleDeletingPhase().catch((err) => { this.logger.error(err) }) @@ -95,9 +99,12 @@ export class DedicatedDatabaseTaskService { return } + const connectionOk = await this.dbService.databaseConnectionIsOk(appid) + manifest = await this.dbService.getDeployManifest(region, user, appid) const unavailable = manifest?.status?.phase !== 'Running' - if (unavailable) { + + if (unavailable && !connectionOk) { await this.relock(appid, waitingTime) return } diff --git a/server/src/database/dedicated-database/dedicated-database.service.ts b/server/src/database/dedicated-database/dedicated-database.service.ts index c9840f9..4af52b5 100644 --- a/server/src/database/dedicated-database/dedicated-database.service.ts +++ b/server/src/database/dedicated-database/dedicated-database.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common' +import { Injectable, Logger } from '@nestjs/common' import { Region } from 'src/region/entities/region' import { DedicatedDatabase, @@ -36,6 +36,8 @@ const p_exec = promisify(exec) @Injectable() export class DedicatedDatabaseService { + private readonly logger = new Logger(DedicatedDatabase.name) + constructor( private readonly cluster: ClusterService, private readonly mongoService: MongoService, @@ -288,7 +290,7 @@ export class DedicatedDatabaseService { `mongodump --uri='${connectionUri}' --gzip --archive=${filePath}`, ) } catch (error) { - console.error(`failed to export db ${appid}`, error) + this.logger.error(`failed to export db ${appid}`, error) throw error } finally { await SystemDatabase.db @@ -387,6 +389,40 @@ export class DedicatedDatabaseService { } } + async databaseConnectionIsOk(appid: string): Promise { + try { + const { client } = await this.findAndConnect(appid) + const admin = client.db('admin') + const replSetStatus = await admin.command({ replSetGetStatus: 1 }) + const members = replSetStatus.members + const replicaSetOk: boolean = replSetStatus.ok === 1 + + const healthyMembers = members.filter( + (member: any) => member.health === 1, + ) + + const primary = healthyMembers.find( + (member: any) => member.stateStr === 'PRIMARY', + ) + + const majorityCount = Math.ceil(members.length / 2) + + const isClusterHealthy = + replicaSetOk && primary && healthyMembers.length >= majorityCount + + if (isClusterHealthy) { + return true + } + + return false + } catch (error) { + this.logger.verbose( + `dedicatedDatabase health check failed ${appid}\n${error.message}`, + ) + return false + } + } + async checkDatabaseSyncLimit(uid: ObjectId) { const count = await SystemDatabase.db .collection('DatabaseSyncRecord')