diff --git a/apps/api/package.json b/apps/api/package.json index 391ab22a9..891374c7c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -54,9 +54,9 @@ "@hono/swagger-ui": "0.2.1", "@hono/zod-openapi": "0.9.5", "@octokit/rest": "^18.12.0", - "@opentelemetry/instrumentation": "^0.54.0", - "@opentelemetry/instrumentation-http": "^0.54.0", - "@opentelemetry/sdk-node": "^0.54.0", + "@opentelemetry/instrumentation": "^0.54.2", + "@opentelemetry/instrumentation-http": "^0.54.2", + "@opentelemetry/sdk-node": "^0.54.2", "@sentry/node": "^7.55.2", "@supercharge/promise-pool": "^3.2.0", "@ucast/core": "^1.10.2", diff --git a/apps/api/src/billing/services/provider-cleanup/provider-cleanup.service.ts b/apps/api/src/billing/services/provider-cleanup/provider-cleanup.service.ts new file mode 100644 index 000000000..32b4255a1 --- /dev/null +++ b/apps/api/src/billing/services/provider-cleanup/provider-cleanup.service.ts @@ -0,0 +1,90 @@ +import { LoggerService } from "@akashnetwork/logging"; +import { singleton } from "tsyringe"; + +import { BillingConfig, InjectBillingConfig } from "@src/billing/providers"; +import { UserWalletOutput, UserWalletRepository } from "@src/billing/repositories"; +import { ManagedUserWalletService, RpcMessageService } from "@src/billing/services"; +import { ErrorService } from "@src/core/services/error/error.service"; +import { ProviderCleanupSummarizer } from "@src/deployment/lib/provider-cleanup-summarizer/provider-cleanup-summarizer"; +import { DeploymentRepository } from "@src/deployment/repositories/deployment/deployment.repository"; +import { TxSignerService } from "../tx-signer/tx-signer.service"; + +export interface ProviderCleanupParams { + concurrency: number; + providerAddress: string; + dryRun: boolean; +} + +@singleton() +export class ProviderCleanupService { + private readonly logger = LoggerService.forContext(ProviderCleanupService.name); + + constructor( + @InjectBillingConfig() private readonly config: BillingConfig, + private readonly userWalletRepository: UserWalletRepository, + private readonly managedUserWalletService: ManagedUserWalletService, + private readonly txSignerService: TxSignerService, + private readonly deploymentRepository: DeploymentRepository, + private readonly rpcMessageService: RpcMessageService, + private readonly errorService: ErrorService + ) {} + + async cleanup(options: ProviderCleanupParams) { + const summary = new ProviderCleanupSummarizer(); + await this.userWalletRepository.paginate({ query: { isTrialing: true }, limit: options.concurrency || 10 }, async wallets => { + const cleanUpAllWallets = wallets.map(async wallet => { + await this.errorService.execWithErrorHandler( + { + wallet, + event: "PROVIDER_CLEAN_UP_ERROR", + context: ProviderCleanupService.name + }, + () => this.cleanUpForWallet(wallet, options, summary) + ); + }); + + await Promise.all(cleanUpAllWallets); + }); + + this.logger.info({ event: "PROVIDER_CLEAN_UP_SUMMARY", summary: summary.summarize(), dryRun: options.dryRun }); + } + + private async cleanUpForWallet(wallet: UserWalletOutput, options: ProviderCleanupParams, summary: ProviderCleanupSummarizer) { + const client = await this.txSignerService.getClientForAddressIndex(wallet.id); + const deployments = await this.deploymentRepository.findDeploymentsForProvider({ + owner: wallet.address, + provider: options.providerAddress + }); + + const closeAllWalletStaleDeployments = deployments.map(async deployment => { + const message = this.rpcMessageService.getCloseDeploymentMsg(wallet.address, deployment.dseq); + this.logger.info({ event: "PROVIDER_CLEAN_UP", params: { owner: wallet.address, dseq: deployment.dseq } }); + + try { + if (!options.dryRun) { + await client.signAndBroadcast([message]); + this.logger.info({ event: "PROVIDER_CLEAN_UP_SUCCESS" }); + } + } catch (error) { + if (error.message.includes("not allowed to pay fees")) { + if (!options.dryRun) { + await this.managedUserWalletService.authorizeSpending({ + address: wallet.address, + limits: { + fees: this.config.FEE_ALLOWANCE_REFILL_AMOUNT + } + }); + await client.signAndBroadcast([message]); + this.logger.info({ event: "PROVIDER_CLEAN_UP_SUCCESS" }); + } + } else { + throw error; + } + } finally { + summary.inc("deploymentCount"); + } + }); + + await Promise.all(closeAllWalletStaleDeployments); + } +} diff --git a/apps/api/src/console.ts b/apps/api/src/console.ts index e2ec98ff6..115bc32b1 100644 --- a/apps/api/src/console.ts +++ b/apps/api/src/console.ts @@ -14,6 +14,7 @@ import { chainDb } from "@src/db/dbConnection"; import { TopUpDeploymentsController } from "@src/deployment/controllers/deployment/deployment.controller"; import { UserController } from "@src/user/controllers/user/user.controller"; import { UserConfigService } from "@src/user/services/user-config/user-config.service"; +import { ProviderController } from "./deployment/controllers/provider/provider.controller"; const program = new Command(); @@ -42,13 +43,25 @@ program program .command("cleanup-stale-deployments") .description("Close deployments without leases created at least 10min ago") - .option("-c, --concurrency ", "How much wallets is processed concurrently", value => z.number({ coerce: true }).optional().default(10).parse(value)) + .option("-c, --concurrency ", "How many wallets is processed concurrently", value => z.number({ coerce: true }).optional().default(10).parse(value)) .action(async (options, command) => { await executeCliHandler(command.name(), async () => { await container.resolve(TopUpDeploymentsController).cleanUpStaleDeployment(options); }); }); +program + .command("cleanup-provider-deployments") + .description("Close trial deployments for a provider") + .option("-c, --concurrency ", "How many wallets is processed concurrently", value => z.number({ coerce: true }).optional().default(10).parse(value)) + .option("-d, --dry-run", "Dry run the trial provider cleanup", false) + .option("-p, --provider ", "Provider address", value => z.string().parse(value)) + .action(async (options, command) => { + await executeCliHandler(command.name(), async () => { + await container.resolve(ProviderController).cleanupProviderDeployments(options); + }); + }); + const userConfig = container.resolve(UserConfigService); program .command("cleanup-stale-anonymous-users") diff --git a/apps/api/src/deployment/controllers/provider/provider.controller.ts b/apps/api/src/deployment/controllers/provider/provider.controller.ts index d2c6261d8..de90d8154 100644 --- a/apps/api/src/deployment/controllers/provider/provider.controller.ts +++ b/apps/api/src/deployment/controllers/provider/provider.controller.ts @@ -1,12 +1,20 @@ import { singleton } from "tsyringe"; +import { ProviderCleanupParams, ProviderCleanupService } from "@src/billing/services/provider-cleanup/provider-cleanup.service"; import { TrialProvidersService } from "@src/deployment/services/trial-providers/trial-providers.service"; @singleton() export class ProviderController { - constructor(private readonly trialProvidersService: TrialProvidersService) {} + constructor( + private readonly trialProvidersService: TrialProvidersService, + private readonly providerCleanupService: ProviderCleanupService + ) {} async getTrialProviders(): Promise { return await this.trialProvidersService.getTrialProviders(); } + + async cleanupProviderDeployments(options: ProviderCleanupParams) { + return await this.providerCleanupService.cleanup(options); + } } diff --git a/apps/api/src/deployment/lib/provider-cleanup-summarizer/provider-cleanup-summarizer.ts b/apps/api/src/deployment/lib/provider-cleanup-summarizer/provider-cleanup-summarizer.ts new file mode 100644 index 000000000..a2e82a84e --- /dev/null +++ b/apps/api/src/deployment/lib/provider-cleanup-summarizer/provider-cleanup-summarizer.ts @@ -0,0 +1,25 @@ +interface ProviderCleanupSummary { + deploymentCount: number; +} + +export class ProviderCleanupSummarizer { + private deploymentCount = 0; + + inc(param: keyof ProviderCleanupSummary, value = 1) { + this[param] += value; + } + + set(param: keyof ProviderCleanupSummary, value: number) { + this[param] = value; + } + + get(param: keyof ProviderCleanupSummary) { + return this[param]; + } + + summarize(): ProviderCleanupSummary { + return { + deploymentCount: this.deploymentCount + }; + } +} diff --git a/apps/api/src/deployment/repositories/deployment/deployment.repository.ts b/apps/api/src/deployment/repositories/deployment/deployment.repository.ts index db10f5c13..047b26a1d 100644 --- a/apps/api/src/deployment/repositories/deployment/deployment.repository.ts +++ b/apps/api/src/deployment/repositories/deployment/deployment.repository.ts @@ -7,6 +7,11 @@ export interface StaleDeploymentsOptions { owner: string; } +export interface ProviderCleanupOptions { + owner: string; + provider: string; +} + export interface StaleDeploymentsOutput { dseq: number; } @@ -37,4 +42,27 @@ export class DeploymentRepository { return deployments ? (deployments as unknown as StaleDeploymentsOutput[]) : []; } + + async findDeploymentsForProvider(options: ProviderCleanupOptions): Promise { + const deployments = await Deployment.findAll({ + attributes: ["dseq"], + where: { + owner: options.owner, + closedHeight: null + }, + include: [ + { + model: Lease, + attributes: [], + required: true, + where: { + provider: options.provider + } + } + ], + raw: true + }); + + return deployments ? (deployments as unknown as StaleDeploymentsOutput[]) : []; + } } diff --git a/apps/provider-console/package.json b/apps/provider-console/package.json index 9f49db74f..5ebc7c4cb 100644 --- a/apps/provider-console/package.json +++ b/apps/provider-console/package.json @@ -22,7 +22,6 @@ "@mui/icons-material": "^5.11.11", "@mui/material": "^5.4.4", "@mui/material-nextjs": "^5.15.11", - "@opentelemetry/instrumentation-lru-memoizer": "^0.41.0", "@radix-ui/react-icons": "^1.3.0", "@sentry/nextjs": "^8.34.0", "@sentry/tracing": "^7.114.0", diff --git a/package-lock.json b/package-lock.json index 0bf63f2f4..9c70de49d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ }, "apps/api": { "name": "@akashnetwork/console-api", - "version": "2.33.1-beta.0", + "version": "2.34.0", "license": "Apache-2.0", "dependencies": { "@akashnetwork/akash-api": "^1.3.0", @@ -54,9 +54,9 @@ "@hono/swagger-ui": "0.2.1", "@hono/zod-openapi": "0.9.5", "@octokit/rest": "^18.12.0", - "@opentelemetry/instrumentation": "^0.54.0", - "@opentelemetry/instrumentation-http": "^0.54.0", - "@opentelemetry/sdk-node": "^0.54.0", + "@opentelemetry/instrumentation": "^0.54.2", + "@opentelemetry/instrumentation-http": "^0.54.2", + "@opentelemetry/sdk-node": "^0.54.2", "@sentry/node": "^7.55.2", "@supercharge/promise-pool": "^3.2.0", "@ucast/core": "^1.10.2", @@ -199,6 +199,7 @@ "version": "0.54.2", "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.54.2.tgz", "integrity": "sha512-go6zpOVoZVztT9r1aPd79Fr3OWiD4N24bCPJsIKkBses8oyFo12F/Ew3UBTdIu6hsW4HC4MVEJygG6TEyJI/lg==", + "license": "Apache-2.0", "dependencies": { "@opentelemetry/api-logs": "0.54.2", "@types/shimmer": "^1.2.0", @@ -228,7 +229,7 @@ }, "apps/deploy-web": { "name": "@akashnetwork/console-web", - "version": "2.24.1-beta.2", + "version": "2.25.0", "license": "Apache-2.0", "dependencies": { "@akashnetwork/akash-api": "^1.3.0", @@ -574,7 +575,6 @@ "@mui/icons-material": "^5.11.11", "@mui/material": "^5.4.4", "@mui/material-nextjs": "^5.15.11", - "@opentelemetry/instrumentation-lru-memoizer": "^0.41.0", "@radix-ui/react-icons": "^1.3.0", "@sentry/nextjs": "^8.34.0", "@sentry/tracing": "^7.114.0", @@ -8967,6 +8967,7 @@ "version": "0.54.2", "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.54.2.tgz", "integrity": "sha512-mABjJ34UcU32pg8g18L9xBh0U3JON/2F6/57BYYy8AZJp2a71lZjcKr0T00pICoic50TW5HvcTrmyfMil+AiXQ==", + "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "1.27.0", "@opentelemetry/instrumentation": "0.54.2", @@ -9148,94 +9149,6 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.41.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.41.0.tgz", - "integrity": "sha512-6OePkk4RYCPVsnS0TroEK6UZzxxxjVWaE6EPdOn2qxGHMtm+Qb80tiBQ6BbmC+f7bjc27O85JY8gxeTybhHZXw==", - "dependencies": { - "@opentelemetry/instrumentation": "^0.54.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer/node_modules/@opentelemetry/api-logs": { - "version": "0.54.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.54.2.tgz", - "integrity": "sha512-4MTVwwmLgUh5QrJnZpYo6YRO5IBLAggf2h8gWDblwRagDStY13aEvt7gGk3jewrMaPlHiF83fENhIx0HO97/cQ==", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer/node_modules/@opentelemetry/instrumentation": { - "version": "0.54.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.54.2.tgz", - "integrity": "sha512-go6zpOVoZVztT9r1aPd79Fr3OWiD4N24bCPJsIKkBses8oyFo12F/Ew3UBTdIu6hsW4HC4MVEJygG6TEyJI/lg==", - "dependencies": { - "@opentelemetry/api-logs": "0.54.2", - "@types/shimmer": "^1.2.0", - "import-in-the-middle": "^1.8.1", - "require-in-the-middle": "^7.1.1", - "semver": "^7.5.2", - "shimmer": "^1.2.1" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.40.0.tgz", - "integrity": "sha512-21xRwZsEdMPnROu/QsaOIODmzw59IYpGFmuC4aFWvMj6stA8+Ei1tX67nkarJttlNjoM94um0N4X26AD7ff54A==", - "dependencies": { - "@opentelemetry/instrumentation": "^0.53.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer/node_modules/@opentelemetry/api-logs": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.53.0.tgz", - "integrity": "sha512-8HArjKx+RaAI8uEIgcORbZIPklyh1YLjPSBus8hjRmvLi6DeFzgOcdZ7KwPabKj8mXF8dX0hyfAyGfycz0DbFw==", - "dependencies": { - "@opentelemetry/api": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer/node_modules/@opentelemetry/instrumentation": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz", - "integrity": "sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A==", - "dependencies": { - "@opentelemetry/api-logs": "0.53.0", - "@types/shimmer": "^1.2.0", - "import-in-the-middle": "^1.8.1", - "require-in-the-middle": "^7.1.1", - "semver": "^7.5.2", - "shimmer": "^1.2.1" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, "node_modules/@opentelemetry/instrumentation-mongodb": { "version": "0.47.0", "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.47.0.tgz", @@ -9855,6 +9768,7 @@ "version": "0.54.2", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.54.2.tgz", "integrity": "sha512-afn8GBpA7Gb55aU0LUxIQ+oe6QxLhsf+Te9iw12Non3ZAspzdoCcfz5+hqecwpuVpEDdnj5iSalF7VVaL2pDeg==", + "license": "Apache-2.0", "dependencies": { "@opentelemetry/api-logs": "0.54.2", "@opentelemetry/core": "1.27.0",