fix: Retry 502-504 errors 3x with backoff

https://harperdb.atlassian.net/browse/STUDIO-116

Author: dawsontoth

src/config/getInstanceClient.ts

@@ -1,6 +1,7 @@
 import { apiClient } from '@/config/apiClient';
 import { authStore, EntityIds, OverallAppSignIn } from '@/features/auth/store/authStore';
 import { rejectReplicationFailures } from '@/integrations/api/replication';
+import { curryRetryGatewayErrors } from '@/integrations/api/retryGatewayErrors';
 import axios from 'axios';
 
 interface InstanceClient {
@@ -45,6 +46,9 @@ export function getInstanceClient({ id = OverallAppSignIn, operationsUrl, port,
 		},
 		baseURL,
 	});
-	client.interceptors.response.use(rejectReplicationFailures);
+	client.interceptors.response.use(
+		rejectReplicationFailures,
+		curryRetryGatewayErrors(client),
+	);
 	return client;
 }

src/integrations/api/retryGatewayErrors.test.ts

@@ -0,0 +1,99 @@
+import { curryRetryGatewayErrors } from '@/integrations/api/retryGatewayErrors';
+import * as sleepModule from '@/lib/sleep';
+import type { AxiosInstance, AxiosRequestConfig } from 'axios';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('@/lib/sleep', () => ({
+	sleep: vi.fn().mockResolvedValue(true),
+}));
+
+describe('curryRetryGatewayErrors', () => {
+	let client: Pick<AxiosInstance, 'request'>;
+
+	beforeEach(() => {
+		vi.clearAllMocks();
+		client = {
+			request: vi.fn().mockResolvedValue({ ok: true }),
+		} as Pick<AxiosInstance, 'request'>;
+	});
+
+	it('should be defined and return a function', () => {
+		const handler = curryRetryGatewayErrors(client);
+		expect(handler).toBeDefined();
+		expect(typeof handler).toBe('function');
+	});
+
+	it('rejects if error has no config', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const err = { response: { status: 502 } } as unknown as { config?: AxiosRequestConfig };
+
+		await expect(handler(err)).rejects.toBe(err);
+		expect(sleepModule.sleep).not.toHaveBeenCalled();
+		expect(client.request).not.toHaveBeenCalled();
+	});
+
+	it('rejects and does not retry for non-gateway status codes', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const config: AxiosRequestConfig & { __retryCount?: number } = {};
+		const err = { response: { status: 500 }, config };
+
+		await expect(handler(err)).rejects.toBe(err);
+		expect(sleepModule.sleep).not.toHaveBeenCalled();
+		expect(client.request).not.toHaveBeenCalled();
+		// Should not mutate retry count
+		expect(config.__retryCount).toBeUndefined();
+	});
+
+	it('retries on 502 and increments __retryCount with initial delay on first retry', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const config: AxiosRequestConfig & { __retryCount?: number } = {};
+		const err = { response: { status: 502 }, config };
+
+		const result = await handler(err);
+
+		expect(sleepModule.sleep).toHaveBeenCalledTimes(1);
+		expect(sleepModule.sleep).toHaveBeenCalledWith(5_000);
+		expect(config.__retryCount).toBe(1);
+		expect(client.request).toHaveBeenCalledTimes(1);
+		expect(client.request).toHaveBeenCalledWith(config);
+		expect(result).toEqual({ ok: true });
+	});
+
+	it('retries on 503 with exponential backoff: more backoff on second retry', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const config: AxiosRequestConfig & { __retryCount?: number } = { __retryCount: 1 };
+		const err = { response: { status: 503 }, config };
+
+		await handler(err);
+
+		expect(sleepModule.sleep).toHaveBeenCalledTimes(1);
+		expect(sleepModule.sleep).toHaveBeenCalledWith(10_000);
+		expect(config.__retryCount).toBe(2);
+		expect(client.request).toHaveBeenCalledTimes(1);
+	});
+
+	it('retries on 504 with the largest backoff on third retry', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const config: AxiosRequestConfig & { __retryCount?: number } = { __retryCount: 2 };
+		const err = { response: { status: 504 }, config };
+
+		await handler(err);
+
+		expect(sleepModule.sleep).toHaveBeenCalledTimes(1);
+		expect(sleepModule.sleep).toHaveBeenCalledWith(20_000);
+		expect(config.__retryCount).toBe(3);
+		expect(client.request).toHaveBeenCalledTimes(1);
+	});
+
+	it('stops retrying when max retries reached (>= 3)', async () => {
+		const handler = curryRetryGatewayErrors(client);
+		const config: AxiosRequestConfig & { __retryCount?: number } = { __retryCount: 3 };
+		const err = { response: { status: 502 }, config };
+
+		await expect(handler(err)).rejects.toBe(err);
+		expect(sleepModule.sleep).not.toHaveBeenCalled();
+		expect(client.request).not.toHaveBeenCalled();
+		// Should not change the retry count when rejecting due to max retries
+		expect(config.__retryCount).toBe(3);
+	});
+});

src/integrations/api/retryGatewayErrors.ts

@@ -0,0 +1,34 @@
+import { sleep } from '@/lib/sleep';
+import { AxiosInstance } from 'axios';
+
+export function curryRetryGatewayErrors(client: Pick<AxiosInstance, 'request'>) {
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	return async (error: any) => {
+		const status = error?.response?.status as number | undefined;
+		const config = error?.config;
+		if (!config) {
+			return Promise.reject(error);
+		}
+
+		const shouldRetry = status === 502 || status === 503 || status === 504;
+		if (!shouldRetry) {
+			return Promise.reject(error);
+		}
+
+		const maxRetries = 3;
+		config.__retryCount = config.__retryCount || 0;
+		if (config.__retryCount >= maxRetries) {
+			return Promise.reject(error);
+		}
+
+		config.__retryCount += 1;
+
+		// 1x, 2x, 4x
+		const retryPower = Math.pow(2, config.__retryCount - 1);
+		// 5s, 10s, 20s
+		const delay = 5_000 * retryPower;
+		await sleep(delay);
+
+		return client.request(config);
+	};
+}