Replace request with axios

Drop in replacement for `request` with `got` is not possible, as `got` no proxy support has. `Axios` comes with proxy support and similar size of package as `got`.

Change-type: major
Signed-off-by: Harald Fischer <[email protected]>
Signed-off-by: fisehara <[email protected]>

package.json

@@ -64,14 +64,14 @@
     "@types/proxy-addr": "^2.0.0",
     "@types/randomstring": "^1.1.8",
     "@types/redlock": "^4.0.4",
-    "@types/request": "^2.48.8",
     "@types/semver": "^7.3.13",
     "@types/tar": "^6.1.4",
     "@types/uuid": "^9.0.1",
     "@types/validator": "^13.7.15",
     "array-sort": "^1.0.0",
     "avsc": "^5.7.7",
     "aws-sdk": "^2.1365.0",
+    "axios": "^1.5.1",
     "balena-device-config": "^6.3.0",
     "balena-semver": "^2.3.2",
     "basic-auth": "^2.0.1",
@@ -107,7 +107,6 @@
     "randomstring": "^1.2.3",
     "rate-limiter-flexible": "^2.4.1",
     "redlock": "^4.2.0",
-    "request": "^2.88.2",
     "rsmq": "^0.12.4",
     "semver": "^7.5.0",
     "snappy": "^7.2.2",
@@ -116,6 +115,7 @@
     "tar": "^6.1.13",
     "thirty-two": "^1.0.2",
     "ts-node": "^10.9.1",
+    "tunnel": "^0.0.6",
     "typed-error": "^3.2.2",
     "typescript": "^5.2.2",
     "uuid": "^9.0.0",

src/features/contracts/contracts-directory.ts

@@ -1,5 +1,4 @@
 import _ from 'lodash';
-import request from 'request';
 import tar from 'tar';
 import glob from 'fast-glob';
 import fs from 'fs';
@@ -12,6 +11,7 @@ import type { RepositoryInfo, Contract } from './index';
 import { getBase64DataUri } from '../../lib/utils';
 import { captureException } from '../../infra/error-handling';
 import { CONTRACT_ALLOWLIST } from '../../lib/config';
+import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
 
 const pipeline = util.promisify(stream.pipeline);
 const exists = util.promisify(fs.exists);
@@ -125,16 +125,16 @@ const prepareContractDirectory = async (repo: RepositoryInfo) => {
 	return archiveDir;
 };
 
-const getRequestOptions = (repo: RepositoryInfo) => {
+const getRequestOptions = (repo: RepositoryInfo): AxiosRequestConfig => {
 	const auth = repo.token
 		? `Basic ${Buffer.from(repo.token).toString('base64')}`
 		: '';
 	return {
-		followRedirect: true,
 		headers: {
 			'User-Agent': 'balena',
 			Authorization: auth,
 		},
+		responseType: 'stream',
 	};
 };
 
@@ -148,22 +148,27 @@ export const fetchContractsLocally = async (repos: RepositoryInfo[]) => {
 			});
 
 			// We cast to ReadableStream explicitly because `request.get is of type `request.Request` and it controls whether it is a readable or writable stream internally so it is not typings-compatible with ReadableStream, even though it it functionally equivalent.
-			const get = request
-				.get(getArchiveLinkForRepo(repo), getRequestOptions(repo))
-				.on('response', function (this: request.Request, response) {
-					if (response.statusCode !== 200) {
-						// On any non-200 responses just error and abort the request
-						this.emit(
-							'error',
-							new Error(
-								`Invalid response while fetching contracts: ${response.statusMessage}`,
-							),
-						);
-						this.abort();
-					}
-				}) as unknown as NodeJS.ReadableStream;
-
-			await pipeline(get, untar);
+			const streamResponse = await axios.get(
+				getArchiveLinkForRepo(repo),
+				getRequestOptions(repo),
+			);
+
+			const readableStream =
+				streamResponse.data as unknown as NodeJS.ReadableStream;
+
+			readableStream.on('data', (response: AxiosResponse) => {
+				if (response.status !== 200) {
+					// On any non-200 responses just error and abort the request
+					readableStream.emit(
+						'error',
+						new Error(
+							`Invalid response while fetching contracts: ${response.statusText}`,
+						),
+					);
+				}
+			});
+
+			await pipeline(readableStream, untar);
 		}),
 	);
 };

src/features/device-proxy/device-proxy.ts

@@ -15,7 +15,8 @@ import {
 	API_VPN_SERVICE_API_KEY,
 	VPN_CONNECT_PROXY_PORT,
 } from '../../lib/config';
-import { requestAsync, RequestResponse } from '../../infra/request-promise';
+import axios, { AxiosResponse as RequestResponse, AxiosInstance } from 'axios';
+import tunnel from 'tunnel';
 import { checkInt, throttledForEach } from '../../lib/utils';
 
 // Degraded network, slow devices, compressed docker binaries and any combination of these factors
@@ -47,24 +48,24 @@ const validateSupervisorResponse = (
 	res: Response,
 	filter: Filter,
 ) => {
-	const [{ statusCode, headers }, body] = response;
+	const { status: statusCode, headers, data } = response;
 	const contentType = headers?.['content-type'];
 	if (contentType != null) {
 		if (/^application\/json/i.test(contentType)) {
 			let jsonBody;
-			if (_.isObject(body)) {
-				jsonBody = body;
+			if (_.isObject(data)) {
+				jsonBody = data;
 			} else {
 				try {
-					jsonBody = JSON.parse(body);
+					jsonBody = JSON.parse(data);
 				} catch (e) {
 					return badSupervisorResponse(req, res, filter, 'Invalid JSON data');
 				}
 			}
 			res.status(statusCode).json(jsonBody);
 		} else if (/^text\/(plain|html)/.test(contentType)) {
-			if (/^([A-Za-z0-9\s:'.?!,/-])*$/g.test(body)) {
-				res.status(statusCode).set('Content-Type', 'text/plain').send(body);
+			if (/^([A-Za-z0-9\s:'.?!,/-])*$/g.test(data)) {
+				res.status(statusCode).set('Content-Type', 'text/plain').send(data);
 			} else {
 				badSupervisorResponse(req, res, filter, 'Invalid TEXT data');
 			}
@@ -82,7 +83,7 @@ const validateSupervisorResponse = (
 };
 
 const multiResponse = (responses: RequestResponse[]) =>
-	responses.map(([response]) => _.pick(response, 'statusCode', 'body'));
+	responses.map((response) => _.pick(response, 'statusCode', 'body'));
 
 export const proxy = async (req: Request, res: Response) => {
 	const filter: Filter = {};
@@ -266,14 +267,24 @@ async function requestDevices({
 				device.api_port || 80
 			}${url}?apikey=${device.api_secret}`;
 			try {
-				return await requestAsync({
-					uri: deviceUrl,
-					json: data,
-					proxy: `http://resin_api:${API_VPN_SERVICE_API_KEY}@${vpnIp}:${VPN_CONNECT_PROXY_PORT}`,
-					tunnel: true,
+				const agent = tunnel.httpsOverHttp({
+					proxy: {
+						host: `http://resin_api:${API_VPN_SERVICE_API_KEY}@${vpnIp}`,
+						port: VPN_CONNECT_PROXY_PORT,
+					},
+				});
+				const axiosClient: AxiosInstance = axios.create({
+					httpsAgent: agent,
+					proxy: false,
+				});
+				const test = await axiosClient({
 					method,
+					data,
+					url: deviceUrl,
 					timeout: DEVICE_REQUEST_TIMEOUT,
 				});
+				console.log(`test:${JSON.stringify(test, null, 2)}`);
+				return test;
 			} catch (err) {
 				if (!wait) {
 					// If we don't care about waiting for the request then we just ignore the error and continue

src/index.ts

@@ -158,7 +158,6 @@ export type { ApplicationType } from './features/application-types/application-t
 export type { DeviceTypeJson } from './features/device-types/device-type-json';
 
 export { DefaultApplicationType } from './features/application-types/application-types';
-export * as request from './infra/request-promise';
 export * as redis from './infra/redis';
 export * as scheduler from './infra/scheduler';
 export * as cache from './infra/cache';

test/test-lib/fileupload-helper.ts

@@ -1,5 +1,5 @@
 import * as fs from 'fs/promises';
-import { requestAsync } from '../../src/infra/request-promise';
+import axios from 'axios';
 import { expect } from 'chai';
 
 export async function checkFileExists(
@@ -10,8 +10,8 @@ export async function checkFileExists(
 	const start = Date.now();
 	while (Date.now() - start < timeout) {
 		try {
-			const [response] = await requestAsync({ url, method: 'GET' });
-			if (response.statusCode !== 200) {
+			const response = await axios(url, { method: 'GET' });
+			if (response.status !== 200) {
 				return false;
 			}
 		} catch (error) {
@@ -23,15 +23,14 @@ export async function checkFileExists(
 }
 
 export async function expectEqualBlobs(url: string, localBlobPath: string) {
-	const [response, fileRes] = await requestAsync({
-		url,
+	const response = await axios(url, {
 		method: 'GET',
-		encoding: null,
+		responseType: 'blob',
 	});
 
-	expect(response.statusCode).to.be.eq(200);
+	expect(response.status).to.be.eq(200);
 
 	const originalFile = await fs.readFile(localBlobPath);
-	const diff = originalFile.compare(fileRes);
+	const diff = originalFile.compare(response.data);
 	expect(diff).to.be.eq(0);
 }

test/test-lib/fixtures.ts

@@ -5,9 +5,8 @@ import _ from 'lodash';
 import path from 'path';
 import { randomUUID } from 'crypto';
 
-import { Headers } from 'request';
 import { API_HOST } from '../../src/lib/config';
-import { requestAsync } from '../../src/infra/request-promise';
+import axios, { AxiosHeaders } from 'axios';
 import { version } from './versions';
 import { supertest } from './supertest';
 
@@ -34,26 +33,22 @@ const createResource = async (args: {
 	user?: { token: string };
 }) => {
 	const { resource, method = 'POST', body = {}, user } = args;
-	const headers: Headers = {};
+	const headers = new AxiosHeaders();
 
 	if (user != null) {
 		headers.Authorization = `Bearer ${user.token}`;
 	}
-
-	const [response, responseBody] = await requestAsync({
+	const { data: responseBody, status: statusCode } = await axios({
 		url: `http://${API_HOST}/${version}/${resource}`,
 		headers,
 		method,
-		json: true,
-		body,
+		data: body,
+		responseEncoding: 'utf-8',
+		responseType: 'json',
 	});
 
-	if (response.statusCode !== 201) {
-		logErrorAndThrow(
-			`Failed to create: ${resource}`,
-			response.statusCode,
-			responseBody,
-		);
+	if (statusCode !== 201) {
+		logErrorAndThrow(`Failed to create: ${resource}`, statusCode, responseBody);
 	}
 
 	return responseBody;