From 969ff30138b12a8dd4ed880e06304d87c0c114c4 Mon Sep 17 00:00:00 2001 From: Harald Fischer Date: Thu, 26 Oct 2023 22:53:59 +0200 Subject: [PATCH] Replace `request` with `axios` Drop in replacement for `request` with `got` is not possible, as `got` no proxy support has. `Axios` comes with proxy support and similar size of package as `got`. Change-type: major Signed-off-by: Harald Fischer --- package-lock.json | 245 ++++++---------------- package.json | 3 + src/features/device-proxy/device-proxy.ts | 37 ++-- src/index.ts | 1 - src/infra/request-promise/index.ts | 54 ----- test/test-lib/fileupload-helper.ts | 15 +- test/test-lib/fixtures.ts | 18 +- 7 files changed, 107 insertions(+), 266 deletions(-) delete mode 100644 src/infra/request-promise/index.ts diff --git a/package-lock.json b/package-lock.json index 22749627f..06e403610 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,11 +43,13 @@ "@types/request": "^2.48.8", "@types/semver": "^7.3.13", "@types/tar": "^6.1.4", + "@types/tunnel": "^0.0.5", "@types/uuid": "^9.0.1", "@types/validator": "^13.7.15", "array-sort": "^1.0.0", "avsc": "^5.7.7", "aws-sdk": "^2.1365.0", + "axios": "^1.5.1", "balena-device-config": "^6.3.0", "balena-semver": "^2.3.2", "basic-auth": "^2.0.1", @@ -92,6 +94,7 @@ "tar": "^6.1.13", "thirty-two": "^1.0.2", "ts-node": "^10.9.1", + "tunnel": "^0.0.6", "typed-error": "^3.2.2", "typescript": "^5.2.2", "uuid": "^9.0.0", @@ -1873,126 +1876,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@napi-rs/snappy-android-arm-eabi": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", - "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-android-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", - "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", - "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", - "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-freebsd-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", - "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", - "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", - "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", - "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@napi-rs/snappy-linux-x64-gnu": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", @@ -2023,51 +1906,6 @@ "node": ">= 10" } }, - "node_modules/@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", - "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", - "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-x64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", - "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3449,6 +3287,14 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.4.tgz", "integrity": "sha512-95Sfz4nvMAb0Nl9DTxN3j64adfwfbBPEYq14VN7zT5J5O2M9V6iZMIIQU1U+pJyl9agHYHNCqhCXgyEtIRRa5A==" }, + "node_modules/@types/tunnel": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.5.tgz", + "integrity": "sha512-XGvAonhX1CeULtu4BTzy5nrsOWbxE2SzOBO1c+T16y4gyNnL7EPX7O5cjISdL2hSxXcuTXVieLJa+rSsKxGp+g==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/uuid": { "version": "9.0.6", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.6.tgz", @@ -4212,6 +4058,29 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, + "node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -6651,6 +6520,25 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -6755,20 +6643,6 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "devOptional": true }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -10097,6 +9971,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -11627,6 +11506,14 @@ "node": ">=0.6.x" } }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index bcf58e16b..98fb1d7c0 100644 --- a/package.json +++ b/package.json @@ -67,11 +67,13 @@ "@types/request": "^2.48.8", "@types/semver": "^7.3.13", "@types/tar": "^6.1.4", + "@types/tunnel": "^0.0.5", "@types/uuid": "^9.0.1", "@types/validator": "^13.7.15", "array-sort": "^1.0.0", "avsc": "^5.7.7", "aws-sdk": "^2.1365.0", + "axios": "^1.5.1", "balena-device-config": "^6.3.0", "balena-semver": "^2.3.2", "basic-auth": "^2.0.1", @@ -116,6 +118,7 @@ "tar": "^6.1.13", "thirty-two": "^1.0.2", "ts-node": "^10.9.1", + "tunnel": "^0.0.6", "typed-error": "^3.2.2", "typescript": "^5.2.2", "uuid": "^9.0.0", diff --git a/src/features/device-proxy/device-proxy.ts b/src/features/device-proxy/device-proxy.ts index f7674625c..7168aa819 100644 --- a/src/features/device-proxy/device-proxy.ts +++ b/src/features/device-proxy/device-proxy.ts @@ -15,7 +15,8 @@ import { API_VPN_SERVICE_API_KEY, VPN_CONNECT_PROXY_PORT, } from '../../lib/config'; -import { requestAsync, RequestResponse } from '../../infra/request-promise'; +import axios, { AxiosResponse as RequestResponse, AxiosInstance } from 'axios'; +import tunnel from 'tunnel'; import { checkInt, throttledForEach } from '../../lib/utils'; // Degraded network, slow devices, compressed docker binaries and any combination of these factors @@ -47,24 +48,24 @@ const validateSupervisorResponse = ( res: Response, filter: Filter, ) => { - const [{ statusCode, headers }, body] = response; + const { status: statusCode, headers, data } = response; const contentType = headers?.['content-type']; if (contentType != null) { if (/^application\/json/i.test(contentType)) { let jsonBody; - if (_.isObject(body)) { - jsonBody = body; + if (_.isObject(data)) { + jsonBody = data; } else { try { - jsonBody = JSON.parse(body); + jsonBody = JSON.parse(data); } catch (e) { return badSupervisorResponse(req, res, filter, 'Invalid JSON data'); } } res.status(statusCode).json(jsonBody); } else if (/^text\/(plain|html)/.test(contentType)) { - if (/^([A-Za-z0-9\s:'.?!,/-])*$/g.test(body)) { - res.status(statusCode).set('Content-Type', 'text/plain').send(body); + if (/^([A-Za-z0-9\s:'.?!,/-])*$/g.test(data)) { + res.status(statusCode).set('Content-Type', 'text/plain').send(data); } else { badSupervisorResponse(req, res, filter, 'Invalid TEXT data'); } @@ -82,7 +83,7 @@ const validateSupervisorResponse = ( }; const multiResponse = (responses: RequestResponse[]) => - responses.map(([response]) => _.pick(response, 'statusCode', 'body')); + responses.map((response) => _.pick(response, 'statusCode', 'body')); export const proxy = async (req: Request, res: Response) => { const filter: Filter = {}; @@ -266,14 +267,24 @@ async function requestDevices({ device.api_port || 80 }${url}?apikey=${device.api_secret}`; try { - return await requestAsync({ - uri: deviceUrl, - json: data, - proxy: `http://resin_api:${API_VPN_SERVICE_API_KEY}@${vpnIp}:${VPN_CONNECT_PROXY_PORT}`, - tunnel: true, + const agent = tunnel.httpsOverHttp({ + proxy: { + host: `http://resin_api:${API_VPN_SERVICE_API_KEY}@${vpnIp}`, + port: VPN_CONNECT_PROXY_PORT, + }, + }); + const axiosClient: AxiosInstance = axios.create({ + httpsAgent: agent, + proxy: false, + }); + const test = await axiosClient({ method, + data, + url: deviceUrl, timeout: DEVICE_REQUEST_TIMEOUT, }); + console.log(`test:${JSON.stringify(test, null, 2)}`); + return test; } catch (err) { if (!wait) { // If we don't care about waiting for the request then we just ignore the error and continue diff --git a/src/index.ts b/src/index.ts index 3617e1e52..3da196327 100644 --- a/src/index.ts +++ b/src/index.ts @@ -158,7 +158,6 @@ export type { ApplicationType } from './features/application-types/application-t export type { DeviceTypeJson } from './features/device-types/device-type-json'; export { DefaultApplicationType } from './features/application-types/application-types'; -export * as request from './infra/request-promise'; export * as redis from './infra/redis'; export * as scheduler from './infra/scheduler'; export * as cache from './infra/cache'; diff --git a/src/infra/request-promise/index.ts b/src/infra/request-promise/index.ts deleted file mode 100644 index 517c7cd2f..000000000 --- a/src/infra/request-promise/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -import Bluebird from 'bluebird'; -import request from 'request'; - -import { EXTERNAL_HTTP_TIMEOUT_MS } from '../../lib/config'; - -type Request = typeof request; - -export type RequestResponse = [request.Response, any]; - -interface PromisifiedRequest extends Request { - getAsync( - options: - | (request.UriOptions & request.CoreOptions) - | (request.UrlOptions & request.CoreOptions), - ): Bluebird; - getAsync( - uri: string, - options?: request.CoreOptions, - ): Bluebird; - - postAsync: ( - arg1: - | (request.UriOptions & request.CoreOptions) - | (request.UrlOptions & request.CoreOptions), - ) => Bluebird; - putAsync: ( - arg1: - | (request.UriOptions & request.CoreOptions) - | (request.UrlOptions & request.CoreOptions), - ) => Bluebird; - delAsync: ( - arg1: - | (request.UriOptions & request.CoreOptions) - | (request.UrlOptions & request.CoreOptions), - ) => Bluebird; -} - -export const defaultRequest = request.defaults({ - timeout: EXTERNAL_HTTP_TIMEOUT_MS, -}); - -const promisifiedRequest = Bluebird.promisifyAll(defaultRequest, { - multiArgs: true, -}) as any as PromisifiedRequest; - -export const requestAsync = Bluebird.promisify(promisifiedRequest, { - multiArgs: true, -}) as any as ( - arg1: - | (request.UriOptions & request.CoreOptions) - | (request.UrlOptions & request.CoreOptions), -) => Bluebird; - -export default promisifiedRequest; diff --git a/test/test-lib/fileupload-helper.ts b/test/test-lib/fileupload-helper.ts index 6cdf9a40b..868b9e010 100644 --- a/test/test-lib/fileupload-helper.ts +++ b/test/test-lib/fileupload-helper.ts @@ -1,5 +1,5 @@ import * as fs from 'fs/promises'; -import { requestAsync } from '../../src/infra/request-promise'; +import axios from 'axios'; import { expect } from 'chai'; export async function checkFileExists( @@ -10,8 +10,8 @@ export async function checkFileExists( const start = Date.now(); while (Date.now() - start < timeout) { try { - const [response] = await requestAsync({ url, method: 'GET' }); - if (response.statusCode !== 200) { + const response = await axios(url, { method: 'GET' }); + if (response.status !== 200) { return false; } } catch (error) { @@ -23,15 +23,14 @@ export async function checkFileExists( } export async function expectEqualBlobs(url: string, localBlobPath: string) { - const [response, fileRes] = await requestAsync({ - url, + const response = await axios(url, { method: 'GET', - encoding: null, + responseType: 'blob', }); - expect(response.statusCode).to.be.eq(200); + expect(response.status).to.be.eq(200); const originalFile = await fs.readFile(localBlobPath); - const diff = originalFile.compare(fileRes); + const diff = originalFile.compare(response.data); expect(diff).to.be.eq(0); } diff --git a/test/test-lib/fixtures.ts b/test/test-lib/fixtures.ts index a3f5136a1..14c28c8ba 100644 --- a/test/test-lib/fixtures.ts +++ b/test/test-lib/fixtures.ts @@ -7,7 +7,7 @@ import { randomUUID } from 'crypto'; import { Headers } from 'request'; import { API_HOST } from '../../src/lib/config'; -import { requestAsync } from '../../src/infra/request-promise'; +import axios from 'axios'; import { version } from './versions'; import { supertest } from './supertest'; @@ -39,21 +39,17 @@ const createResource = async (args: { if (user != null) { headers.Authorization = `Bearer ${user.token}`; } - - const [response, responseBody] = await requestAsync({ + const { data: responseBody, status: statusCode } = await axios({ url: `http://${API_HOST}/${version}/${resource}`, headers, method, - json: true, - body, + data: body, + responseEncoding: 'utf-8', + responseType: 'json', }); - if (response.statusCode !== 201) { - logErrorAndThrow( - `Failed to create: ${resource}`, - response.statusCode, - responseBody, - ); + if (statusCode !== 201) { + logErrorAndThrow(`Failed to create: ${resource}`, statusCode, responseBody); } return responseBody;