feat: custom retries (#107)

* feat: custom retries * fix: retry condition * feat: add retryCondition for http requests --------- Co-authored-by: nataliadskch <[email protected]> Co-authored-by: Roman Fedorenkov <[email protected]>
gravity-ui · Feb 14, 2025 · 6cc9775 · 6cc9775
1 parent d72ddc2
commit 6cc9775
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 5 deletions.
diff --git a/integration-test/client/schema/meta.ts b/integration-test/client/schema/meta.ts
@@ -59,6 +59,12 @@ const actions = {
         action: 'MethodWithError',
         insecure: true,
     },
+    methodWithErrorAndRetries: {
+        ...config,
+        action: 'MethodWithError',
+        insecure: true,
+        retries: 2,
+    },
     methodWithDeadline: {
         ...config,
         action: 'MethodWithDeadline',

diff --git a/integration-test/client/unary.test.ts b/integration-test/client/unary.test.ts
@@ -20,6 +20,10 @@ import {getGatewayControllers} from '../../lib';
 import {ErrorConstructor, createCoreContext} from './create-core-context';
 import {schema} from './schema/meta';
 
+const mockGrpcRetryCondition = jest.fn((error) => {
+    return Boolean(error?.details === 'Error details here');
+});
+
 function getControllers() {
     return getGatewayControllers(
         {local: schema},
@@ -33,6 +37,7 @@ function getControllers() {
             getAuthHeaders: () => undefined,
             proxyHeaders: [],
             withDebugHeaders: false,
+            grpcRetryCondition: mockGrpcRetryCondition,
         },
     );
 }
@@ -80,6 +85,25 @@ describe('Unary requests tests', () => {
         await expectStatsToSendOk();
     });
 
+    it('should correctly handle retries', async () => {
+        await expect(
+            controllers.api.local.meta.methodWithErrorAndRetries(getApiActionConfig()),
+        ).rejects.toMatchObject({
+            debugHeaders: {
+                'x-request-id': requestId,
+            },
+            error: {
+                code: 'GATEWAY_REQUEST_ERROR',
+                status: 500,
+                details: {
+                    grpcCode: 15,
+                },
+            },
+        });
+        await expectStatsToSendError();
+        expect(mockGrpcRetryCondition).toHaveBeenCalledTimes(2);
+    });
+
     it('should correctly rejects when backend error', async () => {
         await expect(
             controllers.api.local.meta.methodWithError(getApiActionConfig()),

diff --git a/lib/components/grpc.ts b/lib/components/grpc.ts
@@ -1045,8 +1045,11 @@ export default function createGrpcAction<Context extends GatewayContext>(
                                     error &&
                                     options.grpcRecreateService &&
                                     isRecreateServiceError(error);
-                                const shouldRetry = error && retries && isRetryableError(error);
-
+                                const shouldRetry =
+                                    error &&
+                                    retries &&
+                                    (options.grpcRetryCondition?.(error) ??
+                                        isRetryableError(error));
                                 if (shouldRecreateService) {
                                     ctx.log(
                                         `Service client for ${config.protoKey} is going to be re-created`,

diff --git a/lib/components/rest.ts b/lib/components/rest.ts
@@ -74,6 +74,7 @@ export default function createRestAction<Context extends GatewayContext>(
     const defaultAxiosClient = getAxiosClient(
         timeout,
         config?.retries,
+        options?.axiosRetryCondition,
         options?.axiosConfig,
         options?.axiosInterceptors,
     );
@@ -282,6 +283,7 @@ export default function createRestAction<Context extends GatewayContext>(
             axiosClient = getAxiosClient(
                 customActionTimeout,
                 config?.retries,
+                options.axiosRetryCondition,
                 customActionAxiosConfig,
                 options?.axiosInterceptors,
             );

diff --git a/lib/index.ts b/lib/index.ts
@@ -107,6 +107,7 @@ function createApiAction<
                 proxyDebugHeaders: config.proxyDebugHeaders,
                 axiosConfig: config.axiosConfig,
                 axiosInterceptors: config.axiosInterceptors,
+                axiosRetryCondition: config.axiosRetryCondition,
                 validationSchema: config.validationSchema,
                 encodePathArgs: config.encodePathArgs,
                 getAuthHeaders: config.getAuthHeaders,
@@ -129,6 +130,7 @@ function createApiAction<
             sendStats: config.sendStats,
             proxyHeaders: config.proxyHeaders,
             proxyDebugHeaders: config.proxyDebugHeaders,
+            grpcRetryCondition: config.grpcRetryCondition,
             grpcOptions: config.grpcOptions,
             grpcRecreateService,
             getAuthHeaders: config.getAuthHeaders,

diff --git a/lib/models/common.ts b/lib/models/common.ts
@@ -1,13 +1,19 @@
 import {IncomingHttpHeaders} from 'http';
 
-import {ClientDuplexStream, ClientReadableStream, ClientWritableStream} from '@grpc/grpc-js';
+import {
+    ClientDuplexStream,
+    ClientReadableStream,
+    ClientWritableStream,
+    type ServiceError,
+} from '@grpc/grpc-js';
 import {HandlerType} from '@grpc/grpc-js/build/src/server-call';
 import {
     AxiosInterceptorManager,
     AxiosRequestConfig,
     AxiosResponse,
     InternalAxiosRequestConfig,
 } from 'axios';
+import {IAxiosRetryConfig} from 'axios-retry';
 import type {Request, Response} from 'express';
 import * as protobufjs from 'protobufjs';
 
@@ -89,6 +95,9 @@ export interface GatewayError {
     requestId?: string;
 }
 
+export type GrpcRetryCondition = (error: ServiceError) => boolean;
+export type AxiosRetryCondition = IAxiosRetryConfig['retryCondition'];
+
 export type ProxyHeadersFunction = (
     headers: IncomingHttpHeaders,
     type: ControllerType,
@@ -114,6 +123,8 @@ export type ResponseContentType = AxiosResponse['headers']['Content-Type'];
 export interface GatewayApiOptions<Context extends GatewayContext> {
     serviceName: string;
     timeout?: number;
+    grpcRetryCondition?: GrpcRetryCondition;
+    axiosRetryCondition?: AxiosRetryCondition;
     sendStats?: SendStats<Context>;
     grpcOptions?: object;
     grpcRecreateService?: boolean;
@@ -446,6 +457,8 @@ export interface GatewayConfig<
     env?: string;
     actions?: string[];
     timeout?: number;
+    grpcRetryCondition?: GrpcRetryCondition;
+    axiosRetryCondition?: AxiosRetryCondition;
     grpcOptions?: object;
     grpcRecreateService?: boolean;
     axiosConfig?: AxiosRequestConfig;

diff --git a/lib/utils/axios.ts b/lib/utils/axios.ts
@@ -1,5 +1,5 @@
 import axios, {AxiosRequestConfig} from 'axios';
-import axiosRetry from 'axios-retry';
+import axiosRetry, {IAxiosRetryConfig} from 'axios-retry';
 import _ from 'lodash';
 
 import {DEFAULT_AXIOS_OPTIONS, DEFAULT_TIMEOUT} from '../constants';
@@ -8,6 +8,7 @@ import {AxiosInterceptorsConfig} from '../models/common';
 export function getAxiosClient(
     timeout: number = DEFAULT_TIMEOUT,
     retries = 0,
+    customRetryCondition?: IAxiosRetryConfig['retryCondition'],
     axiosConfig: AxiosRequestConfig = DEFAULT_AXIOS_OPTIONS,
     {request: reqInterceptors, response: resInterceptors}: AxiosInterceptorsConfig = {},
 ) {
@@ -28,7 +29,10 @@ export function getAxiosClient(
                 return false;
             }
 
-            return axiosRetry.isNetworkError(error) || axiosRetry.isRetryableError(error);
+            return (
+                customRetryCondition?.(error) ??
+                (axiosRetry.isNetworkError(error) || axiosRetry.isRetryableError(error))
+            );
         },
         onRetry: (retryCount, _error, requestConfig) => {
             _.set(requestConfig, ['headers', 'x-request-attempt'], retryCount);