From fb84d691767a81019d8f79143f43563e0b7e49c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 19 Dec 2024 12:18:54 +0100 Subject: [PATCH] chore: check TS in the tests and enable `strictNullChecks` in there (#2776) - adds a new NPM script called `tsc-check-tests` to run `tsc --noEmit` on all the project files including tests (except E2E tests) - runs this new check inside the tests workflow - enables `strictNullChecks` in the tests, since without that some things cannot work (namely the `Result` type in adaptive crawler, generally speaking, the narrowing of discriminated union types) - adds a lot of non-null assertion operators, the tests were initially written in JS and way too many places work with optional properties as if they were not optional - disables `prefer-destructuring` eslint rule, maybe its just my pet peeve, but I hate it --- .eslintrc.json | 2 + .github/workflows/test-ci.yml | 5 ++ package.json | 1 + .../internals/adaptive-playwright-crawler.ts | 2 +- .../browser-plugins/plugins.test.ts | 4 +- test/browser-pool/browser-pool.test.ts | 8 +- test/core/autoscaling/autoscaled_pool.test.ts | 8 +- test/core/autoscaling/snapshotter.test.ts | 16 ++-- .../playwright_launcher.test.ts | 4 +- .../puppeteer_launcher.test.ts | 2 +- .../adaptive_playwright_crawler.test.ts | 13 +-- test/core/crawlers/basic_crawler.test.ts | 24 +++--- test/core/crawlers/browser_crawler.test.ts | 50 +++++------ test/core/crawlers/cheerio_crawler.test.ts | 38 +++++---- test/core/crawlers/dom_crawler.test.ts | 6 +- test/core/crawlers/file_download.test.ts | 2 +- test/core/crawlers/http_crawler.test.ts | 8 +- test/core/crawlers/playwright_crawler.test.ts | 10 +-- test/core/crawlers/puppeteer_crawler.test.ts | 26 +++--- test/core/crawlers/statistics.test.ts | 8 +- .../core/enqueue_links/click_elements.test.ts | 8 +- test/core/enqueue_links/enqueue_links.test.ts | 26 +++--- test/core/error_tracker.test.ts | 4 +- test/core/playwright_utils.test.ts | 4 +- test/core/proxy_configuration.test.ts | 38 ++++----- .../puppeteer_request_interception.test.ts | 6 +- test/core/puppeteer_utils.test.ts | 5 +- test/core/request_list.test.ts | 82 ++++++++++--------- test/core/session_pool/session_pool.test.ts | 6 +- test/core/sitemap_request_list.test.ts | 32 ++++---- test/core/storages/dataset.test.ts | 17 ++-- test/core/storages/key_value_store.test.ts | 24 +++--- test/core/storages/request_queue.test.ts | 44 +++++----- test/core/storages/utils.test.ts | 2 +- .../cheerio-curl-impersonate-ts/actor/main.ts | 4 +- test/shared/MemoryStorageEmulator.ts | 2 +- test/shared/_helper.ts | 6 +- test/tsconfig.json | 4 +- test/utils/cheerio.test.ts | 4 +- test/utils/extract-urls.test.ts | 7 +- test/utils/general.test.ts | 5 +- test/utils/memory-info.test.ts | 10 +-- test/utils/social.test.ts | 78 ++++++++++-------- website/roa-loader/index.js | 4 +- 44 files changed, 350 insertions(+), 309 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 6f62b37f8bff..5f5e2ac905ee 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -45,6 +45,8 @@ } ], "@typescript-eslint/no-empty-interface": "off", + "@typescript-eslint/prefer-destructuring": "off", + "prefer-destructuring": "off", "no-empty-function": "off", "@typescript-eslint/no-empty-function": "off", "@typescript-eslint/no-explicit-any": "off", diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 755e0a65b23d..c234d3a1ab91 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -70,6 +70,11 @@ jobs: env: YARN_IGNORE_NODE: 1 + - name: Test TS + run: yarn tsc-check-tests + env: + YARN_IGNORE_NODE: 1 + - name: Tests run: yarn test env: diff --git a/package.json b/package.json index d1cec1cd8528..46d26f737e2f 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "test": "vitest run --silent", "test:e2e": "node test/e2e/run.mjs", "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 vitest run --silent", + "tsc-check-tests": "tsc --noEmit --project test/tsconfig.json", "coverage": "vitest --coverage", "publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish", "release:next": "yarn build && yarn publish:next", diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts index 89e2695b6efd..82bd77d8b313 100644 --- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts @@ -213,7 +213,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { private renderingTypePredictor: NonNullable; private resultChecker: NonNullable; private resultComparator: NonNullable; - override readonly stats: AdaptivePlaywrightCrawlerStatistics; + declare readonly stats: AdaptivePlaywrightCrawlerStatistics; /** * Default {@apilink Router} instance that will be used if we don't specify any {@apilink AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}. diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.ts index d3693060967b..cf417fa1e79e 100644 --- a/test/browser-pool/browser-plugins/plugins.test.ts +++ b/test/browser-pool/browser-plugins/plugins.test.ts @@ -219,7 +219,7 @@ describe('Plugins', () => { const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); - const text = await response.text(); + const text = await response!.text(); expect(text).toBe('127.0.0.2'); @@ -247,7 +247,7 @@ describe('Plugins', () => { const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); - const text = await response.text(); + const text = await response!.text(); expect(text).toBe('127.0.0.3'); diff --git a/test/browser-pool/browser-pool.test.ts b/test/browser-pool/browser-pool.test.ts index 5d9f641970b7..43d3cd233eed 100644 --- a/test/browser-pool/browser-pool.test.ts +++ b/test/browser-pool/browser-pool.test.ts @@ -637,11 +637,11 @@ describe.each([ ...commonOptions, useFingerprints: true, }); - const oldGet = browserPoolConfig.fingerprintGenerator.getFingerprint; + const oldGet = browserPoolConfig.fingerprintGenerator!.getFingerprint; const mock = vitest.fn((options) => { return oldGet.bind(browserPoolConfig.fingerprintGenerator)(options); }); - browserPoolConfig.fingerprintGenerator.getFingerprint = mock; + browserPoolConfig.fingerprintGenerator!.getFingerprint = mock; const page: Page = await browserPoolConfig.newPage(); await page.close(); @@ -674,11 +674,11 @@ describe.each([ }, }, }); - const oldGet = browserPoolConfig.fingerprintGenerator.getFingerprint; + const oldGet = browserPoolConfig.fingerprintGenerator!.getFingerprint; const mock = vitest.fn((options) => { return oldGet.bind(browserPoolConfig.fingerprintGenerator)(options); }); - browserPoolConfig.fingerprintGenerator.getFingerprint = mock; + browserPoolConfig.fingerprintGenerator!.getFingerprint = mock; const page: Page = await browserPoolConfig.newPageInNewBrowser(); await page.close(); const [options] = mock.mock.calls[0]; diff --git a/test/core/autoscaling/autoscaled_pool.test.ts b/test/core/autoscaling/autoscaled_pool.test.ts index 7855491960e6..44a08564930d 100644 --- a/test/core/autoscaling/autoscaled_pool.test.ts +++ b/test/core/autoscaling/autoscaled_pool.test.ts @@ -25,7 +25,7 @@ describe('AutoscaledPool', () => { } return new Promise((resolve) => { - const item = range.shift(); + const item = range.shift()!; result.push(item); setTimeout(resolve, 5); }); @@ -55,7 +55,7 @@ describe('AutoscaledPool', () => { } return new Promise((resolve) => { - const item = range.shift(); + const item = range.shift()!; result.push(item); setTimeout(resolve, 5); }); @@ -86,7 +86,7 @@ describe('AutoscaledPool', () => { } return new Promise((resolve) => { - const item = range.shift(); + const item = range.shift()!; result.push(item); setTimeout(resolve, 5); }); @@ -418,8 +418,6 @@ describe('AutoscaledPool', () => { if (!aborted) { await pool.abort(); aborted = true; - } else { - return null; } }, isFinishedFunction: async () => { diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.ts index 5976b27702fe..8b5699ec404f 100644 --- a/test/core/autoscaling/snapshotter.test.ts +++ b/test/core/autoscaling/snapshotter.test.ts @@ -22,8 +22,8 @@ describe('Snapshotter', () => { // mock client data const apifyClient = Configuration.getStorageClient(); const oldStats = apifyClient.stats; - apifyClient.stats = {} as never; - apifyClient.stats.rateLimitErrors = [0, 0, 0]; + apifyClient.stats = {} as any; + apifyClient.stats!.rateLimitErrors = [0, 0, 0]; const config = new Configuration({ systemInfoIntervalMillis: 100 }); const snapshotter = new Snapshotter({ config }); @@ -32,7 +32,7 @@ describe('Snapshotter', () => { await snapshotter.start(); await sleep(625); - apifyClient.stats.rateLimitErrors = [0, 0, 2]; + apifyClient.stats!.rateLimitErrors = [0, 0, 2]; await sleep(625); await snapshotter.stop(); @@ -277,19 +277,19 @@ describe('Snapshotter', () => { // mock client data const apifyClient = Configuration.getStorageClient(); const oldStats = apifyClient.stats; - apifyClient.stats = {} as never; - apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + apifyClient.stats = {} as any; + apifyClient.stats!.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; const snapshotter = new Snapshotter({ maxClientErrors: 1 }); // @ts-expect-error Calling protected method snapshotter._snapshotClient(noop); - apifyClient.stats.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0]; + apifyClient.stats!.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0]; // @ts-expect-error Calling protected method snapshotter._snapshotClient(noop); - apifyClient.stats.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0]; + apifyClient.stats!.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0]; // @ts-expect-error Calling protected method snapshotter._snapshotClient(noop); - apifyClient.stats.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0]; + apifyClient.stats!.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0]; // @ts-expect-error Calling protected method snapshotter._snapshotClient(noop); diff --git a/test/core/browser_launchers/playwright_launcher.test.ts b/test/core/browser_launchers/playwright_launcher.test.ts index 377407bfa048..ec12e97a3b6f 100644 --- a/test/core/browser_launchers/playwright_launcher.test.ts +++ b/test/core/browser_launchers/playwright_launcher.test.ts @@ -193,7 +193,7 @@ describe('launchPlaywright()', () => { }); const plugin = launcher.createBrowserPlugin(); - expect(plugin!.launchOptions.executablePath).toEqual(target); + expect(plugin!.launchOptions!.executablePath).toEqual(target); }); test('does not use default when using chrome', () => { @@ -218,7 +218,7 @@ describe('launchPlaywright()', () => { }); const plugin = launcher.createBrowserPlugin(); - expect(plugin.launchOptions.executablePath).toEqual(newPath); + expect(plugin.launchOptions!.executablePath).toEqual(newPath); }); test('works without default path', async () => { diff --git a/test/core/browser_launchers/puppeteer_launcher.test.ts b/test/core/browser_launchers/puppeteer_launcher.test.ts index c55693ec89f0..448fb6710ca4 100644 --- a/test/core/browser_launchers/puppeteer_launcher.test.ts +++ b/test/core/browser_launchers/puppeteer_launcher.test.ts @@ -17,7 +17,7 @@ import type { Browser, Page } from 'puppeteer'; import { runExampleComServer } from '../../shared/_helper'; -let prevEnvHeadless: string; +let prevEnvHeadless: string | undefined; let proxyServer: Server; let proxyPort: number; const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' }; diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts index 2fa4206b029c..5eb4208451dc 100644 --- a/test/core/crawlers/adaptive_playwright_crawler.test.ts +++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts @@ -1,7 +1,7 @@ import type { Server } from 'http'; import type { AddressInfo } from 'net'; -import { KeyValueStore } from '@crawlee/core'; +import { type Dictionary, KeyValueStore } from '@crawlee/core'; import type { AdaptivePlaywrightCrawlerOptions } from '@crawlee/playwright'; import { AdaptivePlaywrightCrawler, RequestList } from '@crawlee/playwright'; import express from 'express'; @@ -180,7 +180,8 @@ describe('AdaptivePlaywrightCrawler', () => { const resultChecker: AdaptivePlaywrightCrawlerOptions['resultChecker'] = vi.fn( (result) => - result.datasetItems.length > 0 && result.datasetItems.every(({ item }) => item.heading?.length > 0), + result.datasetItems.length > 0 && + result.datasetItems.every(({ item }: Dictionary) => item.heading?.length > 0), ); const crawler = await makeOneshotCrawler( @@ -264,7 +265,7 @@ describe('AdaptivePlaywrightCrawler', () => { await crawler.run(); const state = await localStorageEmulator.getState(); - expect(state.value).toEqual({ count: 3 }); + expect(state!.value).toEqual({ count: 3 }); }); test('should persist key-value store changes', async () => { @@ -297,9 +298,9 @@ describe('AdaptivePlaywrightCrawler', () => { await crawler.run(); const store = localStorageEmulator.getKeyValueStore(); - expect((await store.getRecord('1')).value).toEqual({ content: 42 }); - expect((await store.getRecord('2')).value).toEqual({ content: 42 }); - expect((await store.getRecord('3')).value).toEqual({ content: 42 }); + expect((await store.getRecord('1'))!.value).toEqual({ content: 42 }); + expect((await store.getRecord('2'))!.value).toEqual({ content: 42 }); + expect((await store.getRecord('3'))!.value).toEqual({ content: 42 }); }); test('should not allow direct key-value store manipulation', async () => { diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 3a9879cd479a..aa988e540900 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -102,7 +102,7 @@ describe('BasicCrawler', () => { await basicCrawler.run(); - expect(basicCrawler.autoscaledPool.minConcurrency).toBe(25); + expect(basicCrawler.autoscaledPool!.minConcurrency).toBe(25); expect(processed).toEqual(sourcesCopy); expect(await requestList.isFinished()).toBe(true); expect(await requestList.isEmpty()).toBe(true); @@ -146,12 +146,12 @@ describe('BasicCrawler', () => { const collectResults = (crawler: BasicCrawler): typeof shorthandOptions | typeof autoscaledPoolOptions => { return { - minConcurrency: crawler.autoscaledPool.minConcurrency, - maxConcurrency: crawler.autoscaledPool.maxConcurrency, + minConcurrency: crawler.autoscaledPool!.minConcurrency, + maxConcurrency: crawler.autoscaledPool!.maxConcurrency, // eslint-disable-next-line dot-notation -- accessing a private member - maxRequestsPerMinute: crawler.autoscaledPool['maxTasksPerMinute'], + maxRequestsPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'], // eslint-disable-next-line dot-notation - maxTasksPerMinute: crawler.autoscaledPool['maxTasksPerMinute'], + maxTasksPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'], }; }; @@ -222,7 +222,7 @@ describe('BasicCrawler', () => { async (event) => { const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` })); - let persistResolve: (value?: unknown) => void; + let persistResolve!: (value?: unknown) => void; const persistPromise = new Promise((res) => { persistResolve = res; }); @@ -269,7 +269,7 @@ describe('BasicCrawler', () => { // clean up // @ts-expect-error Accessing private method - await basicCrawler.autoscaledPool._destroy(); + await basicCrawler.autoscaledPool!._destroy(); }, ); @@ -579,7 +579,7 @@ describe('BasicCrawler', () => { await crawler.run(); - expect(request.retryCount).toBe(0); + expect(request!.retryCount).toBe(0); }); test('should crash on CriticalError', async () => { @@ -769,7 +769,7 @@ describe('BasicCrawler', () => { const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished'); - requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()); + requestQueue.fetchNextRequest = async () => queue.pop()!; requestQueue.isEmpty = async () => Promise.resolve(!queue.length); setTimeout(() => queue.push(request0), 10); @@ -818,7 +818,7 @@ describe('BasicCrawler', () => { const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished'); - requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()); + requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!); requestQueue.isEmpty = async () => Promise.resolve(!queue.length); setTimeout(() => queue.push(request0), 10); @@ -1184,8 +1184,8 @@ describe('BasicCrawler', () => { persistStateKey: 'POOL', }, requestHandler: async ({ session }) => { - expect(session.constructor.name).toEqual('Session'); - expect(session.id).toBeDefined(); + expect(session!.constructor.name).toEqual('Session'); + expect(session!.id).toBeDefined(); }, failedRequestHandler: async ({ request }) => { results.push(request); diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts index e70f590c59ff..6c740c7bce49 100644 --- a/test/core/crawlers/browser_crawler.test.ts +++ b/test/core/crawlers/browser_crawler.test.ts @@ -33,7 +33,7 @@ describe('BrowserCrawler', () => { let server: Server; beforeAll(async () => { - prevEnvHeadless = process.env.CRAWLEE_HEADLESS; + prevEnvHeadless = process.env.CRAWLEE_HEADLESS!; process.env.CRAWLEE_HEADLESS = '1'; logLevel = log.getLevel(); log.setLevel(log.LEVELS.ERROR); @@ -48,7 +48,7 @@ describe('BrowserCrawler', () => { }); afterEach(async () => { - puppeteerPlugin = null; + puppeteerPlugin = null as any; }); afterAll(async () => { @@ -74,7 +74,7 @@ describe('BrowserCrawler', () => { const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => { await page.waitForSelector('title'); - expect(response.status()).toBe(200); + expect(response!.status()).toBe(200); request.userData.title = await page.title(); processed.push(request); }; @@ -94,7 +94,7 @@ describe('BrowserCrawler', () => { await browserCrawler.run(); - expect(browserCrawler.autoscaledPool.minConcurrency).toBe(1); + expect(browserCrawler.autoscaledPool!.minConcurrency).toBe(1); expect(processed).toHaveLength(6); expect(failed).toHaveLength(0); @@ -128,13 +128,13 @@ describe('BrowserCrawler', () => { sources: [{ url: 'http://example.com/?q=1' }], }); class TimeoutError extends Error {} - let sessionGoto: Session; + let sessionGoto!: Session; const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( ctx: PuppeteerCrawlingContext, ): Promise { - vitest.spyOn(ctx.session, 'markBad'); - sessionGoto = ctx.session; + vitest.spyOn(ctx.session!, 'markBad'); + sessionGoto = ctx.session!; throw new TimeoutError(); } })({ @@ -308,14 +308,14 @@ describe('BrowserCrawler', () => { maxRequestRetries: 0, preNavigationHooks: [ async (_crawlingContext, gotoOptions) => { - gotoOptions.timeout = 60000; + gotoOptions!.timeout = 60000; }, ], }); await browserCrawler.run(); - expect(optionsGoto.timeout).toEqual(60000); + expect(optionsGoto!.timeout).toEqual(60000); }); test('should ignore errors in Page.close()', async () => { @@ -440,7 +440,7 @@ describe('BrowserCrawler', () => { useSessionPool: true, persistCookiesPerSession: true, requestHandler: async ({ session, request }) => { - loadedCookies.push(session.getCookieString(request.url)); + loadedCookies.push(session!.getCookieString(request.url)); return Promise.resolve(); }, preNavigationHooks: [ @@ -645,7 +645,7 @@ describe('BrowserCrawler', () => { resolve(); called = true; }); - ctx.session.retire(); + ctx.session!.retire(); return ctx.page.goto(ctx.request.url); } })({ @@ -691,7 +691,7 @@ describe('BrowserCrawler', () => { }); describe('proxy', () => { - let requestList: RequestList; + let requestList: RequestList | undefined; beforeEach(async () => { requestList = await RequestList.open({ sources: [ @@ -704,7 +704,7 @@ describe('BrowserCrawler', () => { }); afterEach(() => { - requestList = null; + requestList = undefined; }); // TODO move to actor sdk tests before splitting the repos @@ -812,13 +812,13 @@ describe('BrowserCrawler', () => { }); browserCrawler.browserPool.postLaunchHooks.push((_pageId, browserController) => { - browserProxies.push(browserController.launchContext.proxyUrl); + browserProxies.push(browserController.launchContext.proxyUrl!); }); await browserCrawler.run(); // @ts-expect-error Accessing private property - const proxiesToUse = proxyConfiguration.proxyUrls; + const proxiesToUse = proxyConfiguration.proxyUrls!; for (const proxyUrl of proxiesToUse) { expect(browserProxies.includes(new URL(proxyUrl).href.slice(0, -1))).toBeTruthy(); } @@ -838,9 +838,9 @@ describe('BrowserCrawler', () => { ctx: PuppeteerCrawlingContext, ): Promise { const { session } = ctx; - const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id); + const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); - if (proxyInfo.url !== goodProxyUrl) { + if (proxyInfo!.url !== goodProxyUrl) { throw new Error('ERR_PROXY_CONNECTION_FAILED'); } @@ -859,7 +859,7 @@ describe('BrowserCrawler', () => { }); await expect(browserCrawler.run()).resolves.not.toThrow(); - expect(requestHandler).toHaveBeenCalledTimes(requestList.length()); + expect(requestHandler).toHaveBeenCalledTimes(requestList!.length()); }); test('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => { @@ -871,17 +871,17 @@ describe('BrowserCrawler', () => { /** * The first increment is the base case when the proxy is retrieved for the first time. */ - let numberOfRotations = -requestList.length(); + let numberOfRotations = -requestList!.length(); const browserCrawler = new (class extends BrowserCrawlerTest { protected override async _navigationHandler( ctx: PuppeteerCrawlingContext, ): Promise { const { session } = ctx; - const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id); + const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); numberOfRotations++; - if (proxyInfo.url.includes('localhost')) { + if (proxyInfo!.url.includes('localhost')) { throw new Error('ERR_PROXY_CONNECTION_FAILED'); } @@ -900,8 +900,8 @@ describe('BrowserCrawler', () => { }); await browserCrawler.run(); - expect(failedRequestHandler).toBeCalledTimes(requestList.length()); - expect(numberOfRotations).toBe(requestList.length() * 5); + expect(failedRequestHandler).toBeCalledTimes(requestList!.length()); + expect(numberOfRotations).toBe(requestList!.length() * 5); }); test('proxy rotation logs the original proxy error', async () => { @@ -915,9 +915,9 @@ describe('BrowserCrawler', () => { ctx: PuppeteerCrawlingContext, ): Promise { const { session } = ctx; - const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id); + const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id); - if (proxyInfo.url.includes('localhost')) { + if (proxyInfo!.url.includes('localhost')) { throw new Error(proxyError); } diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts index fc8635bb1b83..05df9809d081 100644 --- a/test/core/crawlers/cheerio_crawler.test.ts +++ b/test/core/crawlers/cheerio_crawler.test.ts @@ -17,6 +17,7 @@ import { } from '@crawlee/cheerio'; import { sleep } from '@crawlee/utils'; import type { Dictionary } from '@crawlee/utils'; +// @ts-expect-error type import of ESM only package import type { OptionsInit } from 'got-scraping'; import iconv from 'iconv-lite'; import { runExampleComServer, responseSamples } from 'test/shared/_helper'; @@ -102,7 +103,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); - expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2); + expect(cheerioCrawler.autoscaledPool!.minConcurrency).toBe(2); expect(processed).toHaveLength(4); expect(failed).toHaveLength(0); @@ -135,7 +136,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); - expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2); + expect(cheerioCrawler.autoscaledPool!.minConcurrency).toBe(2); expect(processed).toHaveLength(4); expect(failed).toHaveLength(0); @@ -171,7 +172,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); - expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2); + expect(cheerioCrawler.autoscaledPool!.minConcurrency).toBe(2); expect(processed).toHaveLength(4); expect(failed).toHaveLength(0); @@ -407,8 +408,8 @@ describe('CheerioCrawler', () => { expect(headers).toHaveLength(4); headers.forEach((h) => { const acceptHeader = h.accept || h.Accept; - expect(acceptHeader.includes('text/html')).toBe(true); - expect(acceptHeader.includes('application/xhtml+xml')).toBe(true); + expect(acceptHeader!.includes('text/html')).toBe(true); + expect(acceptHeader!.includes('application/xhtml+xml')).toBe(true); }); }); @@ -554,7 +555,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); - expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2); + expect(cheerioCrawler.autoscaledPool!.minConcurrency).toBe(2); expect(failed).toHaveLength(0); }); @@ -575,7 +576,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); - expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2); + expect(cheerioCrawler.autoscaledPool!.minConcurrency).toBe(2); expect(failed).toHaveLength(4); }); @@ -720,7 +721,7 @@ describe('CheerioCrawler', () => { const crawler = new CheerioCrawler({ requestList, requestHandler: ({ proxyInfo }) => { - proxies.push(proxyInfo.url); + proxies.push(proxyInfo!.url); }, proxyConfiguration, }); @@ -742,8 +743,8 @@ describe('CheerioCrawler', () => { const proxies: ProxyInfo[] = []; const sessions: Session[] = []; const requestHandler = ({ session, proxyInfo }: CheerioCrawlingContext) => { - proxies.push(proxyInfo); - sessions.push(session); + proxies.push(proxyInfo!); + sessions.push(session!); }; const requestList = await getRequestListForMirror(); @@ -914,7 +915,7 @@ describe('CheerioCrawler', () => { await cheerioCrawler.run(); // @ts-expect-error private symbol - const { sessions } = cheerioCrawler.sessionPool; + const sessions = cheerioCrawler.sessionPool!.sessions; expect(sessions.length).toBe(4); sessions.forEach((session) => { // TODO this test is flaky in CI and we need some more info to debug why. @@ -947,7 +948,7 @@ describe('CheerioCrawler', () => { persistCookiesPerSession: false, maxRequestRetries: 0, requestHandler: ({ session }) => { - sessions.push(session); + sessions.push(session!); }, failedRequestHandler: ({ request }) => { failed.push(request); @@ -1017,7 +1018,7 @@ describe('CheerioCrawler', () => { await crawler.run(); requests.forEach((_req, i) => { if (i >= 1) { - const response = JSON.parse(_req.payload); + const response = JSON.parse(_req.payload!); expect(response.headers['set-cookie']).toEqual(cookie); } @@ -1085,7 +1086,7 @@ describe('CheerioCrawler', () => { }, preNavigationHooks: [ ({ request }) => { - request.headers.Cookie = 'foo=override; coo=kie'; + request.headers!.Cookie = 'foo=override; coo=kie'; }, ], }); @@ -1115,7 +1116,7 @@ describe('CheerioCrawler', () => { }, preNavigationHooks: [ ({ request }) => { - request.headers.Cookie = 'foo=override; coo=kie'; + request.headers!.Cookie = 'foo=override; coo=kie'; }, ], }); @@ -1192,7 +1193,7 @@ describe('CheerioCrawler', () => { const oldHandleRequestF = cheerioCrawler._runRequestHandler; // @ts-expect-error Overriding private method cheerioCrawler._runRequestHandler = async (opts) => { - usedSession = opts.session; + usedSession = opts.session!; return oldHandleRequestF.call(cheerioCrawler, opts); }; @@ -1202,7 +1203,10 @@ describe('CheerioCrawler', () => { // localhost proxy causes proxy errors, session rotations and finally throws, but we don't care } - expect(newUrlSpy).toBeCalledWith(usedSession.id, expect.objectContaining({ request: expect.any(Request) })); + expect(newUrlSpy).toBeCalledWith( + usedSession!.id, + expect.objectContaining({ request: expect.any(Request) }), + ); }); }); diff --git a/test/core/crawlers/dom_crawler.test.ts b/test/core/crawlers/dom_crawler.test.ts index 1549ce9c5ffb..0f027ec816a6 100644 --- a/test/core/crawlers/dom_crawler.test.ts +++ b/test/core/crawlers/dom_crawler.test.ts @@ -16,8 +16,8 @@ let url: string; beforeAll(async () => { server = http.createServer((request, response) => { try { - const requestUrl = new URL(request.url, 'http://localhost'); - router.get(requestUrl.pathname)(request, response); + const requestUrl = new URL(request.url!, 'http://localhost'); + router.get(requestUrl.pathname)!(request, response); } catch (error) { response.destroy(); } @@ -51,7 +51,7 @@ test('works', async () => { const crawler = new JSDOMCrawler({ maxRequestRetries: 0, requestHandler: ({ window }) => { - results.push(window.document.title, window.document.querySelector('p').textContent); + results.push(window.document.title, window.document.querySelector('p')!.textContent!); }, }); diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.ts index 9293efe6fc8a..9a6083890ef0 100644 --- a/test/core/crawlers/file_download.test.ts +++ b/test/core/crawlers/file_download.test.ts @@ -105,7 +105,7 @@ test('streamHandler works', async () => { const crawler = new FileDownload({ maxRequestRetries: 0, streamHandler: async ({ stream }) => { - for await (const chunk of stream as ReadableStream) { + for await (const chunk of stream as unknown as ReadableStream) { result = Buffer.concat([result, chunk]); } }, diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts index 7d6d052f8adf..139217a73809 100644 --- a/test/core/crawlers/http_crawler.test.ts +++ b/test/core/crawlers/http_crawler.test.ts @@ -65,8 +65,8 @@ let url: string; beforeAll(async () => { server = http.createServer((request, response) => { try { - const requestUrl = new URL(request.url, 'http://localhost'); - router.get(requestUrl.pathname)(request, response); + const requestUrl = new URL(request.url!, 'http://localhost'); + router.get(requestUrl.pathname)!(request, response); } catch (error) { response.destroy(); } @@ -301,7 +301,7 @@ test('should ignore http error status codes set by user', async () => { await crawler.run([`${url}/500Error`]); - expect(crawler.autoscaledPool.minConcurrency).toBe(2); + expect(crawler.autoscaledPool!.minConcurrency).toBe(2); expect(failed).toHaveLength(0); }); @@ -320,7 +320,7 @@ test('should throw an error on http error status codes set by user', async () => await crawler.run([`${url}/hello.html`]); - expect(crawler.autoscaledPool.minConcurrency).toBe(2); + expect(crawler.autoscaledPool!.minConcurrency).toBe(2); expect(failed).toHaveLength(1); }); diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts index e48d125c3df6..1f9bbcacf67a 100644 --- a/test/core/crawlers/playwright_crawler.test.ts +++ b/test/core/crawlers/playwright_crawler.test.ts @@ -14,7 +14,7 @@ import { startExpressAppPromise } from '../../shared/_helper'; if (os.platform() === 'win32') vitest.setConfig({ testTimeout: 2 * 60 * 1e3 }); describe('PlaywrightCrawler', () => { - let prevEnvHeadless: string; + let prevEnvHeadless: string | undefined; let logLevel: number; const localStorageEmulator = new MemoryStorageEmulator(); let requestList: RequestList; @@ -82,10 +82,10 @@ describe('PlaywrightCrawler', () => { useState, }: Parameters[0]) => { const state = await useState([]); - expect(response.status()).toBe(200); + expect(response!.status()).toBe(200); request.userData.title = await page.title(); processed.push(request); - expect(response.request().headers()['user-agent']).not.toMatch(/headless/i); + expect(response!.request().headers()['user-agent']).not.toMatch(/headless/i); // firefox now also returns `webdriver: true` since playwright 1.45, we are masking this via fingerprints, // but this test has them disabled, so we can check the default handling (= there is non-default UA even without them) @@ -110,7 +110,7 @@ describe('PlaywrightCrawler', () => { await playwrightCrawler.run(); - expect(playwrightCrawler.autoscaledPool.minConcurrency).toBe(1); + expect(playwrightCrawler.autoscaledPool!.minConcurrency).toBe(1); expect(processed).toHaveLength(6); expect(failed).toHaveLength(0); @@ -138,7 +138,7 @@ describe('PlaywrightCrawler', () => { }); await playwrightCrawler.run(); - expect(options.timeout).toEqual(timeoutSecs * 1000); + expect(options!.timeout).toEqual(timeoutSecs * 1000); }); test('shallow clones browserPoolOptions before normalization', () => { diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts index 814da19bcfb5..3703bbb71419 100644 --- a/test/core/crawlers/puppeteer_crawler.test.ts +++ b/test/core/crawlers/puppeteer_crawler.test.ts @@ -26,7 +26,7 @@ describe('PuppeteerCrawler', () => { let proxyConfiguration: ProxyConfiguration; beforeAll(async () => { - prevEnvHeadless = process.env.CRAWLEE_HEADLESS; + prevEnvHeadless = process.env.CRAWLEE_HEADLESS!; process.env.CRAWLEE_HEADLESS = '1'; logLevel = log.getLevel(); log.setLevel(log.LEVELS.ERROR); @@ -93,11 +93,11 @@ describe('PuppeteerCrawler', () => { const requestListLarge = await RequestList.open({ sources: sourcesLarge }); const requestHandler = async ({ page, request, response }: PuppeteerCrawlingContext) => { await page.waitForSelector('title'); - asserts.push(response.status() === 200); + asserts.push(response!.status() === 200); request.userData.title = await page.title(); processed.push(request); asserts.push( - !response + !response! .request() .headers() ['user-agent'].match(/headless/i), @@ -118,7 +118,7 @@ describe('PuppeteerCrawler', () => { await puppeteerCrawler.run(); - expect(puppeteerCrawler.autoscaledPool.minConcurrency).toBe(1); + expect(puppeteerCrawler.autoscaledPool!.minConcurrency).toBe(1); expect(processed).toHaveLength(6); expect(failed).toHaveLength(0); @@ -149,7 +149,7 @@ describe('PuppeteerCrawler', () => { }); await puppeteerCrawler.run(); - expect(options.timeout).toEqual(timeoutSecs * 1000); + expect(options!.timeout).toEqual(timeoutSecs * 1000); }); test('should throw if launchOptions.proxyUrl is supplied', async () => { @@ -246,14 +246,14 @@ describe('PuppeteerCrawler', () => { await requestQueue.drop(); expect(requestHandler).not.toBeCalled(); - const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1].retryCount]); + const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1]!.retryCount]); expect(warnings).toEqual([ ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 1], ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 2], ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 3], ]); - const errors = logErrorSpy.mock.calls.map((call) => [call[0], call[1].retryCount]); + const errors = logErrorSpy.mock.calls.map((call) => [call[0], call[1]!.retryCount]); expect(errors).toEqual([ ['Request failed and reached maximum retries. Navigation timed out after 0.005 seconds.', undefined], ]); @@ -289,14 +289,14 @@ describe('PuppeteerCrawler', () => { await requestQueue.drop(); expect(requestHandler).not.toBeCalled(); - const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1].retryCount]); + const warnings = logWarningSpy.mock.calls.map((call) => [call[0].split('\n')[0], call[1]!.retryCount]); expect(warnings).toEqual([ ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 1], ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 2], ['Reclaiming failed request back to the list or queue. Navigation timed out after 0.005 seconds.', 3], ]); - const errors = logErrorSpy.mock.calls.map((call) => [call[0], call[1].retryCount]); + const errors = logErrorSpy.mock.calls.map((call) => [call[0], call[1]!.retryCount]); expect(errors).toEqual([ ['Request failed and reached maximum retries. Navigation timed out after 0.005 seconds.', undefined], ]); @@ -328,7 +328,7 @@ describe('PuppeteerCrawler', () => { }, requestHandler: async ({ page, session }) => { pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; ')); - sessionCookies = session.getCookieString(serverUrl); + sessionCookies = session!.getCookieString(serverUrl); }, }); @@ -359,8 +359,8 @@ describe('PuppeteerCrawler', () => { }, proxyConfiguration, requestHandler: async ({ proxyInfo, session }) => { - proxies.add(proxyInfo.url); - sessions.add(session.id); + proxies.add(proxyInfo!.url); + sessions.add(session!.id); }, }); @@ -409,7 +409,7 @@ describe('PuppeteerCrawler', () => { browserPoolOptions: { prePageCreateHooks: [ (_id, _controller, options) => { - options.proxyBypassList = ['<-loopback>']; + options!.proxyBypassList = ['<-loopback>']; }, ], }, diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts index 94a919ac81b3..5066f66b001c 100644 --- a/test/core/crawlers/statistics.test.ts +++ b/test/core/crawlers/statistics.test.ts @@ -22,7 +22,7 @@ describe('Statistics', () => { afterEach(async () => { events.off(EventType.PERSIST_STATE); - stats = null; + stats = null as any; }); afterAll(async () => { @@ -281,11 +281,11 @@ describe('Statistics', () => { }); test('should regularly log stats', async () => { - const logged: [string, Dictionary?][] = []; + const logged: [string, Dictionary | undefined | null][] = []; // @ts-expect-error Accessing private prop const infoSpy = vitest.spyOn(stats.log, 'info'); - infoSpy.mockImplementation((...args: [message: string, data?: Record]) => { - logged.push(args); + infoSpy.mockImplementation((message: string, data?: Record | null) => { + logged.push([message, data]); }); stats.startJob(0); diff --git a/test/core/enqueue_links/click_elements.test.ts b/test/core/enqueue_links/click_elements.test.ts index 07f407d46b15..50ec682bf58c 100644 --- a/test/core/enqueue_links/click_elements.test.ts +++ b/test/core/enqueue_links/click_elements.test.ts @@ -110,7 +110,7 @@ testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => { }); test('accepts forefront option', async () => { - const addedRequests: { request: Source; options: RequestQueueOperationOptions }[] = []; + const addedRequests: { request: Source; options?: RequestQueueOperationOptions }[] = []; const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() }); requestQueue.addRequests = async (requests, options) => { addedRequests.push(...requests.map((request) => ({ request, options }))); @@ -136,8 +136,8 @@ testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => { forefront: true, }); expect(addedRequests).toHaveLength(2); - expect(addedRequests[0].options.forefront).toBe(true); - expect(addedRequests[1].options.forefront).toBe(true); + expect(addedRequests[0].options!.forefront).toBe(true); + expect(addedRequests[1].options!.forefront).toBe(true); }); describe('clickElements()', () => { @@ -331,7 +331,7 @@ testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => { await clickElements.clickElements(page, 'textarea', { clickCount: 3, delay: 100 }); expect( await page.evaluate(() => { - const textarea = document.querySelector('textarea'); + const textarea = document.querySelector('textarea')!; return textarea.value.substring(textarea.selectionStart, textarea.selectionEnd); }), ).toBe(text); diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts index cd072b2f106d..fa872160a2f5 100644 --- a/test/core/enqueue_links/enqueue_links.test.ts +++ b/test/core/enqueue_links/enqueue_links.test.ts @@ -79,8 +79,8 @@ describe('enqueueLinks()', () => { afterEach(async () => { if (browser) await browser.close(); - page = null; - browser = null; + page = null!; + browser = null!; }); test('works with item limit', async () => { @@ -389,6 +389,7 @@ describe('enqueueLinks()', () => { const { enqueued, requestQueue } = createRequestQueueMock(); await expect( browserCrawlerEnqueueLinks({ + // @ts-expect-error invalid input options: { selector: '.click', pseudoUrls: null }, page, requestQueue, @@ -431,6 +432,7 @@ describe('enqueueLinks()', () => { await expect( browserCrawlerEnqueueLinks({ + // @ts-expect-error invalid input options: { selector: '.click', pseudoUrls }, page, requestQueue, @@ -541,7 +543,7 @@ describe('enqueueLinks()', () => { if (/example\.com/.test(request.url)) { request.method = 'POST'; } else if (/cool\.com/.test(request.url)) { - request.userData.foo = 'bar'; + request.userData!.foo = 'bar'; } return request; }, @@ -563,7 +565,7 @@ describe('enqueueLinks()', () => { expect(enqueued[2].url).toBe('http://cool.com/'); expect(enqueued[2].method).toBe('GET'); - expect(enqueued[2].userData.foo).toBe('bar'); + expect(enqueued[2].userData!.foo).toBe('bar'); }); }); @@ -575,7 +577,7 @@ describe('enqueueLinks()', () => { }); afterEach(async () => { - $ = null; + $ = null!; }); test('works with globs', async () => { @@ -761,6 +763,7 @@ describe('enqueueLinks()', () => { const { enqueued, requestQueue } = createRequestQueueMock(); await expect( cheerioCrawlerEnqueueLinks({ + // @ts-expect-error invalid input options: { selector: '.click', pseudoUrls: null }, $, requestQueue, @@ -803,6 +806,7 @@ describe('enqueueLinks()', () => { await expect( cheerioCrawlerEnqueueLinks({ + // @ts-expect-error invalid input options: { selector: '.click', pseudoUrls }, $, requestQueue, @@ -935,7 +939,7 @@ describe('enqueueLinks()', () => { if (/example\.com/.test(request.url)) { request.method = 'POST'; } else if (/cool\.com/.test(request.url)) { - request.userData.foo = 'bar'; + request.userData!.foo = 'bar'; } return request; }, @@ -957,11 +961,11 @@ describe('enqueueLinks()', () => { expect(enqueued[2].url).toBe('http://cool.com/'); expect(enqueued[2].method).toBe('GET'); - expect(enqueued[2].userData.foo).toBe('bar'); + expect(enqueued[2].userData!.foo).toBe('bar'); }); test('accepts forefront option', async () => { - const enqueued: { request: Source; options: RequestQueueOperationOptions }[] = []; + const enqueued: { request: Source; options?: RequestQueueOperationOptions }[] = []; const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient }); requestQueue.addRequests = async (requests, options) => { @@ -983,12 +987,12 @@ describe('enqueueLinks()', () => { expect(enqueued).toHaveLength(5); for (let i = 0; i < 5; i++) { - expect(enqueued[i].options.forefront).toBe(true); + expect(enqueued[i].options!.forefront).toBe(true); } }); test('accepts waitForAllRequestsToBeAdded option', async () => { - const enqueued: { request: string | Source; options: AddRequestsBatchedOptions }[] = []; + const enqueued: { request: string | Source; options?: AddRequestsBatchedOptions }[] = []; const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient }); requestQueue.addRequestsBatched = async (requests, options) => { @@ -1010,7 +1014,7 @@ describe('enqueueLinks()', () => { expect(enqueued).toHaveLength(5); for (let i = 0; i < 5; i++) { - expect(enqueued[i].options.waitForAllRequestsToBeAdded).toBe(true); + expect(enqueued[i].options!.waitForAllRequestsToBeAdded).toBe(true); } }); }); diff --git a/test/core/error_tracker.test.ts b/test/core/error_tracker.test.ts index 4d5bba5b950e..b5e9dcc26057 100644 --- a/test/core/error_tracker.test.ts +++ b/test/core/error_tracker.test.ts @@ -371,7 +371,7 @@ test('can shorten the message to the first line', () => { expect(tracker.result).toMatchObject({ 'myscript.js:10:3': { // source - [e.code]: { + [e.code!]: { // code [e.name]: { // name @@ -404,7 +404,7 @@ test('supports error.cause', () => { expect(tracker.result).toMatchObject({ 'myscript.js:10:3': { // source - [e.code]: { + [e.code!]: { // code [e.name]: { // name diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts index 936718a68956..477f877dadc9 100644 --- a/test/core/playwright_utils.test.ts +++ b/test/core/playwright_utils.test.ts @@ -178,7 +178,7 @@ describe('playwrightUtils', () => { }); describe('blockRequests()', () => { - let browser: Browser = null; + let browser: Browser = null as any; beforeAll(async () => { browser = await launchPlaywright(launchContext); }); @@ -235,7 +235,7 @@ describe('playwrightUtils', () => { const response = await playwrightUtils.gotoExtended(page, request); - const { method, headers, bodyLength } = JSON.parse(await response.text()); + const { method, headers, bodyLength } = JSON.parse(await response!.text()); expect(method).toBe('POST'); expect(bodyLength).toBe(16); expect(headers['content-type']).toBe('application/json; charset=utf-8'); diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts index d44468022f0d..e70344109488 100644 --- a/test/core/proxy_configuration.test.ts +++ b/test/core/proxy_configuration.test.ts @@ -64,7 +64,7 @@ describe('ProxyConfiguration', () => { 'http://proxy.com:6666', ]; const newUrlFunction = () => { - return customUrls.pop(); + return customUrls.pop() ?? null; }; const proxyConfiguration = new ProxyConfiguration({ newUrlFunction, @@ -76,9 +76,9 @@ describe('ProxyConfiguration', () => { expect(await proxyConfiguration.newUrl()).toEqual('http://proxy.com:4444'); // through newProxyInfo() - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:3333'); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:2222'); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:1111'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:3333'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:2222'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:1111'); }); test('async newUrlFunction should work correctly', async () => { @@ -92,7 +92,7 @@ describe('ProxyConfiguration', () => { ]; const newUrlFunction = async () => { await new Promise((r) => setTimeout(r, 5)); - return customUrls.pop(); + return customUrls.pop() ?? null; }; const proxyConfiguration = new ProxyConfiguration({ newUrlFunction, @@ -104,9 +104,9 @@ describe('ProxyConfiguration', () => { expect(await proxyConfiguration.newUrl()).toEqual('http://proxy.com:4444'); // through newProxyInfo() - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:3333'); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:2222'); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual('http://proxy.com:1111'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:3333'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:2222'); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual('http://proxy.com:1111'); }); describe('With proxyUrls options', () => { @@ -116,7 +116,7 @@ describe('ProxyConfiguration', () => { }); // @ts-expect-error private property - const { proxyUrls } = proxyConfiguration; + const proxyUrls = proxyConfiguration.proxyUrls!; expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[0]); expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[1]); expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[2]); @@ -131,13 +131,13 @@ describe('ProxyConfiguration', () => { }); // @ts-expect-error TODO private property? - const { proxyUrls } = proxyConfiguration; - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[0]); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[1]); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[2]); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[0]); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[1]); - expect((await proxyConfiguration.newProxyInfo()).url).toEqual(proxyUrls[2]); + const proxyUrls = proxyConfiguration.proxyUrls!; + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[0]); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[1]); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[2]); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[0]); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[1]); + expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[2]); }); test('should rotate custom URLs with sessions correctly', async () => { @@ -147,7 +147,7 @@ describe('ProxyConfiguration', () => { }); // @ts-expect-error TODO private property? - const { proxyUrls } = proxyConfiguration; + const proxyUrls = proxyConfiguration.proxyUrls!; // should use same proxy URL expect(await proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]); expect(await proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]); @@ -214,7 +214,7 @@ describe('ProxyConfiguration', () => { }); // @ts-expect-error protected property - const { tieredProxyUrls } = proxyConfiguration; + const tieredProxyUrls = proxyConfiguration.tieredProxyUrls!; expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[0][0]); expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[0][1]); expect(await proxyConfiguration.newUrl()).toEqual(tieredProxyUrls[1][0]); @@ -232,7 +232,7 @@ describe('ProxyConfiguration', () => { }); // @ts-expect-error protected property - const { tieredProxyUrls } = proxyConfiguration; + const tieredProxyUrls = proxyConfiguration.tieredProxyUrls!; expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[0][0]); expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[1][0]); expect(await proxyConfiguration.newUrl('session-id', { request })).toEqual(tieredProxyUrls[2][0]); diff --git a/test/core/puppeteer_request_interception.test.ts b/test/core/puppeteer_request_interception.test.ts index ad7c4b1246bc..aa008fa099bb 100644 --- a/test/core/puppeteer_request_interception.test.ts +++ b/test/core/puppeteer_request_interception.test.ts @@ -131,7 +131,7 @@ describe('utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandl // Check response that it's correct. const response = await page.goto(`${serverAddress}/special/getDebug`, { waitUntil: 'networkidle0' }); - const { method, headers, bodyLength } = JSON.parse(await response.text()); + const { method, headers, bodyLength } = JSON.parse(await response!.text()); expect(method).toBe('POST'); expect(bodyLength).toBe(16); expect(headers['content-type']).toBe('application/json; charset=utf-8'); @@ -155,7 +155,7 @@ describe('utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandl // Check response that it's correct. const response = await page.goto(`${serverAddress}/special/getDebug`, { waitUntil: 'networkidle0' }); - const { method } = JSON.parse(await response.text()); + const { method } = JSON.parse(await response!.text()); expect(method).toBe('POST'); } finally { await browser.close(); @@ -181,7 +181,7 @@ describe('utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandl }); const response = await page.goto(`${serverAddress}/special/getRawHeaders`); - const rawHeadersArr = JSON.parse(await response.text()) as string[]; + const rawHeadersArr = JSON.parse(await response!.text()) as string[]; const acceptIndex = rawHeadersArr.findIndex((headerItem) => headerItem === 'Accept'); expect(typeof acceptIndex).toBe('number'); diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts index e46a7239ded2..fb7f151722e6 100644 --- a/test/core/puppeteer_utils.test.ts +++ b/test/core/puppeteer_utils.test.ts @@ -209,7 +209,7 @@ describe('puppeteerUtils', () => { }); describe('blockRequests()', () => { - let browser: Browser = null; + let browser: Browser = null as any; beforeAll(async () => { browser = await launchPuppeteer(launchContext); }); @@ -325,6 +325,7 @@ describe('puppeteerUtils', () => { await testRuleType(0); // @ts-expect-error await testRuleType(1); + // @ts-expect-error await testRuleType(null); // @ts-expect-error await testRuleType([]); @@ -377,7 +378,7 @@ describe('puppeteerUtils', () => { const response = await puppeteerUtils.gotoExtended(page, request, { waitUntil: 'networkidle' }); - const { method, headers, bodyLength } = JSON.parse(await response.text()); + const { method, headers, bodyLength } = JSON.parse(await response!.text()); expect(method).toBe('POST'); expect(bodyLength).toBe(16); expect(headers['content-type']).toBe('application/json; charset=utf-8'); diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.ts index e99dc9f2fe76..c94f6bd39045 100644 --- a/test/core/request_list.test.ts +++ b/test/core/request_list.test.ts @@ -9,7 +9,9 @@ import { RequestList, } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; +import type { gotScraping } from '@crawlee/utils'; import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator'; +import { beforeAll, type MockedFunction } from 'vitest'; /** * Stand-in for underscore.js shuffle (weird, but how else?) @@ -29,9 +31,13 @@ vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => { }; }); -const { gotScraping } = await import('@crawlee/utils/src/internals/gotScraping'); +let gotScrapingSpy: MockedFunction; -const gotScrapingSpy = vitest.mocked(gotScraping); +beforeAll(async () => { + // @ts-ignore for some reason, this fails when the project is not built :/ + const { gotScraping } = await import('@crawlee/utils'); + gotScrapingSpy = vitest.mocked(gotScraping); +}); describe('RequestList', () => { let ll: number; @@ -63,12 +69,12 @@ describe('RequestList', () => { const req = await requestList.fetchNextRequest(); - expect(req.url).toBe('https://example.com/1'); + expect(req!.url).toBe('https://example.com/1'); expect(await requestList.isEmpty()).toBe(true); expect(await requestList.isFinished()).toBe(false); expect(await requestList.fetchNextRequest()).toBe(null); - await requestList.markRequestHandled(req); + await requestList.markRequestHandled(req!); expect(await requestList.isEmpty()).toBe(true); expect(await requestList.isFinished()).toBe(true); @@ -119,10 +125,10 @@ describe('RequestList', () => { const r5 = await originalList.fetchNextRequest(); // 5 await originalList.fetchNextRequest(); // 6 - await originalList.markRequestHandled(r1); - await originalList.markRequestHandled(r2); - await originalList.markRequestHandled(r4); - await originalList.reclaimRequest(r5); + await originalList.markRequestHandled(r1!); + await originalList.markRequestHandled(r2!); + await originalList.markRequestHandled(r4!); + await originalList.reclaimRequest(r5!); const newList = await RequestList.open({ sources: sourcesCopy, @@ -130,11 +136,11 @@ describe('RequestList', () => { }); expect(await newList.isEmpty()).toBe(false); - expect((await newList.fetchNextRequest()).url).toBe('https://example.com/3'); - expect((await newList.fetchNextRequest()).url).toBe('https://example.com/5'); - expect((await newList.fetchNextRequest()).url).toBe('https://example.com/6'); - expect((await newList.fetchNextRequest()).url).toBe('https://example.com/7'); - expect((await newList.fetchNextRequest()).url).toBe('https://example.com/8'); + expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/3'); + expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/5'); + expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/6'); + expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/7'); + expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/8'); expect(await newList.isEmpty()).toBe(true); }); @@ -292,11 +298,11 @@ describe('RequestList', () => { const request4 = await requestList.fetchNextRequest(); const request5 = await requestList.fetchNextRequest(); - expect(request1.url).toBe('https://example.com/1'); - expect(request2.url).toBe('https://example.com/2'); - expect(request3.url).toBe('https://example.com/3'); - expect(request4.url).toBe('https://example.com/4'); - expect(request5.url).toBe('https://example.com/5'); + expect(request1!.url).toBe('https://example.com/1'); + expect(request2!.url).toBe('https://example.com/2'); + expect(request3!.url).toBe('https://example.com/3'); + expect(request4!.url).toBe('https://example.com/4'); + expect(request5!.url).toBe('https://example.com/5'); expect(requestList.getState()).toEqual({ inProgress: [ 'https://example.com/1', @@ -318,10 +324,10 @@ describe('RequestList', () => { // Reclaim 3rd 4th // - await requestList.markRequestHandled(request1); - await requestList.markRequestHandled(request2); - await requestList.reclaimRequest(request3); - await requestList.reclaimRequest(request4); + await requestList.markRequestHandled(request1!); + await requestList.markRequestHandled(request2!); + await requestList.reclaimRequest(request3!); + await requestList.reclaimRequest(request4!); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/3', 'https://example.com/4', 'https://example.com/5'], @@ -336,7 +342,7 @@ describe('RequestList', () => { // Mark 5th handled // - await requestList.markRequestHandled(request5); + await requestList.markRequestHandled(request5!); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/3', 'https://example.com/4'], @@ -353,10 +359,10 @@ describe('RequestList', () => { // const reclaimed3 = await requestList.fetchNextRequest(); - expect(reclaimed3.url).toBe('https://example.com/3'); + expect(reclaimed3!.url).toBe('https://example.com/3'); const reclaimed4 = await requestList.fetchNextRequest(); - expect(reclaimed4.url).toBe('https://example.com/4'); - await requestList.markRequestHandled(request4); + expect(reclaimed4!.url).toBe('https://example.com/4'); + await requestList.markRequestHandled(request4!); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/3'], @@ -371,7 +377,7 @@ describe('RequestList', () => { // Mark 3rd handled // - await requestList.markRequestHandled(request3); + await requestList.markRequestHandled(request3!); expect(requestList.getState()).toEqual({ inProgress: [], @@ -388,7 +394,7 @@ describe('RequestList', () => { const request6 = await requestList.fetchNextRequest(); - expect(request6.url).toBe('https://example.com/6'); + expect(request6!.url).toBe('https://example.com/6'); expect(await requestList.fetchNextRequest()).toBe(null); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/6'], @@ -403,7 +409,7 @@ describe('RequestList', () => { // Reclaim 6th // - await requestList.reclaimRequest(request6); + await requestList.reclaimRequest(request6!); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/6'], @@ -420,7 +426,7 @@ describe('RequestList', () => { const reclaimed6 = await requestList.fetchNextRequest(); - expect(reclaimed6.url).toBe('https://example.com/6'); + expect(reclaimed6!.url).toBe('https://example.com/6'); expect(requestList.getState()).toEqual({ inProgress: ['https://example.com/6'], nextIndex: 6, @@ -434,7 +440,7 @@ describe('RequestList', () => { // Mark 6th handled // - await requestList.markRequestHandled(reclaimed6); + await requestList.markRequestHandled(reclaimed6!); expect(requestList.getState()).toEqual({ inProgress: [], @@ -479,7 +485,7 @@ describe('RequestList', () => { // Do some other changes and persist it again. const request2 = await requestList.fetchNextRequest(); expect(requestList.isStatePersisted).toBe(false); - await requestList.markRequestHandled(request2); + await requestList.markRequestHandled(request2!); expect(requestList.isStatePersisted).toBe(false); setValueSpy.mockResolvedValueOnce(); events.emit(EventType.PERSIST_STATE); @@ -487,7 +493,7 @@ describe('RequestList', () => { expect(requestList.isStatePersisted).toBe(true); // Reclaim event doesn't change the state. - await requestList.reclaimRequest(request1); + await requestList.reclaimRequest(request1!); expect(requestList.isStatePersisted).toBe(true); // Now initiate new request list from saved state and check that it's same as state @@ -541,7 +547,7 @@ describe('RequestList', () => { const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue'); const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue'); const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls'); - let persistedRequests; + let persistedRequests: any; const opts = { sources: [ @@ -641,13 +647,13 @@ describe('RequestList', () => { const req3 = await requestList.fetchNextRequest(); expect(requestList.handledCount()).toBe(0); - await requestList.markRequestHandled(req2); + await requestList.markRequestHandled(req2!); expect(requestList.handledCount()).toBe(1); - await requestList.markRequestHandled(req3); + await requestList.markRequestHandled(req3!); expect(requestList.handledCount()).toBe(2); - await requestList.reclaimRequest(req1); + await requestList.reclaimRequest(req1!); expect(requestList.handledCount()).toBe(2); }); @@ -768,7 +774,7 @@ describe('RequestList', () => { const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue'); const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue'); - const name: string = null; + const name: string | null = null; const sources = [{ url: 'https://example.com' }]; const requests = sources.map(({ url }) => ({ url, uniqueKey: url })); diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts index 0f2430e0535b..74c4e7e25fc4 100644 --- a/test/core/session_pool/session_pool.test.ts +++ b/test/core/session_pool/session_pool.test.ts @@ -166,12 +166,12 @@ describe('SessionPool - testing session pool', () => { await sessionPool.persistState(); const kvStore = await KeyValueStore.open(); - // @ts-expect-error private symbol const sessionPoolSaved = await kvStore.getValue>( + // @ts-expect-error private symbol sessionPool.persistStateKey, ); - entries(sessionPoolSaved).forEach(([key, value]) => { + entries(sessionPoolSaved!).forEach(([key, value]) => { if (key !== 'sessions') { expect(value).toEqual(sessionPool[key]); } @@ -180,7 +180,7 @@ describe('SessionPool - testing session pool', () => { // @ts-expect-error private symbol expect(sessionPoolSaved.sessions.length).toEqual(sessionPool.sessions.length); - sessionPoolSaved.sessions.forEach((session, index) => { + sessionPoolSaved!.sessions.forEach((session, index) => { entries(session).forEach(([key, value]) => { // @ts-expect-error private symbol if (sessionPool.sessions[index][key] instanceof Date) { diff --git a/test/core/sitemap_request_list.test.ts b/test/core/sitemap_request_list.test.ts index 5948f9958127..2fe420f292dd 100644 --- a/test/core/sitemap_request_list.test.ts +++ b/test/core/sitemap_request_list.test.ts @@ -3,7 +3,7 @@ import type { AddressInfo } from 'net'; import { Readable } from 'stream'; import { finished } from 'stream/promises'; -import { SitemapRequestList } from '@crawlee/core'; +import { SitemapRequestList, type Request } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import express from 'express'; import { startExpressAppPromise } from 'test/shared/_helper'; @@ -328,8 +328,8 @@ describe('SitemapRequestList', () => { while (!(await list.isEmpty())) { const request = await list.fetchNextRequest(); - firstBatch.push(request); - await list.markRequestHandled(request); + firstBatch.push(request!); + await list.markRequestHandled(request!); } expect(firstBatch).toHaveLength(2); @@ -342,8 +342,8 @@ describe('SitemapRequestList', () => { while (!(await list.isEmpty())) { const request = await list.fetchNextRequest(); - secondBatch.push(request); - await list.markRequestHandled(request); + secondBatch.push(request!); + await list.markRequestHandled(request!); } expect(secondBatch).toHaveLength(5); @@ -430,8 +430,8 @@ describe('SitemapRequestList', () => { while (!(await list.isFinished())) { const request = await list.fetchNextRequest(); - await list.markRequestHandled(request); - requests.push(request); + await list.markRequestHandled(request!); + requests.push(request!); } await expect(list.isEmpty()).resolves.toBe(true); @@ -457,10 +457,10 @@ describe('SitemapRequestList', () => { const request = await list.fetchNextRequest(); if (counter % 2 === 0) { - await list.markRequestHandled(request); - requests.push(request); + await list.markRequestHandled(request!); + requests.push(request!); } else { - await list.reclaimRequest(request); + await list.reclaimRequest(request!); } counter += 1; @@ -485,7 +485,7 @@ describe('SitemapRequestList', () => { const list = await SitemapRequestList.open(options); const firstRequest = await list.fetchNextRequest(); - await list.markRequestHandled(firstRequest); + await list.markRequestHandled(firstRequest!); await list.persistState(); @@ -494,7 +494,7 @@ describe('SitemapRequestList', () => { while (!(await newList.isFinished())) { const request = await newList.fetchNextRequest(); - await newList.markRequestHandled(request); + await newList.markRequestHandled(request!); } expect(list.handledCount()).toBe(1); @@ -526,8 +526,8 @@ describe('SitemapRequestList', () => { const list = await SitemapRequestList.open(options); const firstRequest = await list.fetchNextRequest(); - firstRequest.userData = userDataPayload; - firstLoadedUrl = firstRequest.url; + firstRequest!.userData = userDataPayload; + firstLoadedUrl = firstRequest!.url; await list.persistState(); // simulates a migration in the middle of request processing @@ -536,7 +536,7 @@ describe('SitemapRequestList', () => { const newList = await SitemapRequestList.open(options); const restoredRequest = await newList.fetchNextRequest(); - expect(restoredRequest.url).toEqual(firstLoadedUrl); - expect(restoredRequest.userData).toEqual(userDataPayload); + expect(restoredRequest!.url).toEqual(firstLoadedUrl); + expect(restoredRequest!.userData).toEqual(userDataPayload); }); }); diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts index 98a0ff78e79c..495ea68190f3 100644 --- a/test/core/storages/dataset.test.ts +++ b/test/core/storages/dataset.test.ts @@ -31,14 +31,14 @@ describe('dataset', () => { const pushItemSpy = vitest.spyOn(dataset.client, 'pushItems'); - const mockPushItems = pushItemSpy.mockResolvedValueOnce(null); + const mockPushItems = pushItemSpy.mockResolvedValueOnce(undefined); await dataset.pushData({ foo: 'bar' }); expect(mockPushItems).toBeCalledTimes(1); expect(mockPushItems).toBeCalledWith(JSON.stringify({ foo: 'bar' })); - const mockPushItems2 = pushItemSpy.mockResolvedValueOnce(null); + const mockPushItems2 = pushItemSpy.mockResolvedValueOnce(undefined); await dataset.pushData([{ foo: 'hotel;' }, { foo: 'restaurant' }]); @@ -62,8 +62,8 @@ describe('dataset', () => { }); const mockPushItems = vitest.spyOn(dataset.client, 'pushItems'); - mockPushItems.mockResolvedValueOnce(null); - mockPushItems.mockResolvedValueOnce(null); + mockPushItems.mockResolvedValueOnce(undefined); + mockPushItems.mockResolvedValueOnce(undefined); await dataset.pushData([{ foo: half }, { bar: half }]); @@ -86,8 +86,8 @@ describe('dataset', () => { }); const mockPushItems = vitest.spyOn(dataset.client, 'pushItems'); - mockPushItems.mockResolvedValueOnce(null); - mockPushItems.mockResolvedValueOnce(null); + mockPushItems.mockResolvedValueOnce(undefined); + mockPushItems.mockResolvedValueOnce(undefined); await dataset.pushData(data); @@ -296,6 +296,7 @@ describe('dataset', () => { item.index = index; item.bar = 'xxx'; + // @ts-expect-error FIXME the inference is broken for `reduce()` method return memo.concat(item); }, [], @@ -322,6 +323,7 @@ describe('dataset', () => { item.index = index; item.bar = 'xxx'; + // @ts-expect-error FIXME the inference is broken for `reduce()` method return Promise.resolve(memo.concat(item)); }, [], @@ -367,8 +369,10 @@ describe('dataset', () => { const calledForIndexes: number[] = []; const result = await dataset.reduce( + // @ts-expect-error FIXME the inference is broken for `reduce()` method async (memo, item, index) => { calledForIndexes.push(index); + // @ts-expect-error FIXME the inference is broken for `reduce()` method return Promise.resolve(memo.foo > item.foo ? memo : item); }, undefined, @@ -387,6 +391,7 @@ describe('dataset', () => { offset: 2, }); + // @ts-expect-error FIXME the inference is broken for `reduce()` method expect(result.foo).toBe(5); expect(calledForIndexes).toEqual([1, 2, 3]); }); diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts index 65f3d977b277..0e3a3acab314 100644 --- a/test/core/storages/key_value_store.test.ts +++ b/test/core/storages/key_value_store.test.ts @@ -35,7 +35,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); await store.setValue('key-1', record); @@ -78,7 +78,7 @@ describe('KeyValueStore', () => { const mockDeleteRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'deleteRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); await store.setValue('key-1', null); @@ -112,6 +112,7 @@ describe('KeyValueStore', () => { await expect(store.getValue({})).rejects.toThrow( 'Expected argument to be of type `string` but received type `Object`', ); + // @ts-expect-error JS-side validation await expect(store.getValue(null)).rejects.toThrow( 'Expected argument to be of type `string` but received type `null`', ); @@ -149,6 +150,7 @@ describe('KeyValueStore', () => { await expect(store.recordExists({})).rejects.toThrow( 'Expected argument to be of type `string` but received type `Object`', ); + // @ts-expect-error JS-side validation await expect(store.recordExists(null)).rejects.toThrow( 'Expected argument to be of type `string` but received type `null`', ); @@ -262,7 +264,7 @@ describe('KeyValueStore', () => { // test max length const longKey = 'X'.repeat(257); const err = `The "key" argument "${longKey}" must be at most 256 characters`; - await expect(store.setValue(longKey)).rejects.toThrow(err); + await expect(store.setValue(longKey, '...')).rejects.toThrow(err); }); test('correctly adds charset to content type', async () => { @@ -274,7 +276,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' }); @@ -298,7 +300,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); await store.setValue('key-1', record); @@ -319,7 +321,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' }); @@ -340,7 +342,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); const value = Buffer.from('some text value'); await store.setValue('key-1', value, { contentType: 'image/jpeg; charset=something' }); @@ -362,7 +364,7 @@ describe('KeyValueStore', () => { const mockSetRecord = vitest // @ts-expect-error Accessing private property .spyOn(store.client, 'setRecord') - .mockResolvedValueOnce(null); + .mockResolvedValueOnce(undefined); const value = new PassThrough(); await store.setValue('key-1', value, { contentType: 'plain/text' }); @@ -398,7 +400,7 @@ describe('KeyValueStore', () => { describe('maybeStringify()', () => { test('should work', () => { - expect(maybeStringify({ foo: 'bar' }, { contentType: null })).toBe('{\n "foo": "bar"\n}'); + expect(maybeStringify({ foo: 'bar' }, { contentType: null as any })).toBe('{\n "foo": "bar"\n}'); expect(maybeStringify({ foo: 'bar' }, { contentType: undefined })).toBe('{\n "foo": "bar"\n}'); expect(maybeStringify('xxx', { contentType: undefined })).toBe('"xxx"'); @@ -406,7 +408,7 @@ describe('KeyValueStore', () => { const obj = {} as Dictionary; obj.self = obj; - expect(() => maybeStringify(obj, { contentType: null })).toThrowError( + expect(() => maybeStringify(obj, { contentType: null as any })).toThrowError( 'The "value" parameter cannot be stringified to JSON: Converting circular structure to JSON', ); }); @@ -473,7 +475,7 @@ describe('KeyValueStore', () => { mockListKeys.mockResolvedValueOnce({ isTruncated: false, exclusiveStartKey: 'key0', - nextExclusiveStartKey: null, + nextExclusiveStartKey: undefined, items: [{ key: 'key5', size: 5 }], count: 1, limit: 1, diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts index 085428d806cb..5d3b3ba5a278 100644 --- a/test/core/storages/request_queue.test.ts +++ b/test/core/storages/request_queue.test.ts @@ -11,6 +11,8 @@ import { ProxyConfiguration, } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; +import type { gotScraping } from '@crawlee/utils'; +import type { MockedFunction } from 'vitest'; import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator'; @@ -20,9 +22,13 @@ vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => { }; }); -const { gotScraping } = await import('@crawlee/utils/src/internals/gotScraping'); +let gotScrapingSpy: MockedFunction; -const gotScrapingSpy = vitest.mocked(gotScraping); +beforeAll(async () => { + // @ts-ignore for some reason, this fails when the project is not built :/ + const { gotScraping } = await import('@crawlee/utils'); + gotScrapingSpy = vitest.mocked(gotScraping); +}); describe('RequestQueue remote', () => { const storageClient = Configuration.getStorageClient(); @@ -97,8 +103,8 @@ describe('RequestQueue remote', () => { ), ); - // getRequest() returns null if object was not found. - mockGetRequest.mockResolvedValueOnce(null); + // getRequest() returns undefined if object was not found. + mockGetRequest.mockResolvedValueOnce(undefined); const requestXFromQueue = await queue.getRequest('non-existent'); expect(mockGetRequest).toBeCalledTimes(2); @@ -116,7 +122,7 @@ describe('RequestQueue remote', () => { request: requestBFromQueue, }); - await queue.reclaimRequest(requestBFromQueue, { forefront: true }); + await queue.reclaimRequest(requestBFromQueue!, { forefront: true }); expect(mockUpdateRequest).toBeCalledTimes(1); expect(mockUpdateRequest).toHaveBeenLastCalledWith(requestBFromQueue, { forefront: true }); @@ -148,7 +154,7 @@ describe('RequestQueue remote', () => { request: requestBFromQueue, }); - await queue.markRequestHandled(requestBFromQueue); + await queue.markRequestHandled(requestBFromQueue!); expect(mockUpdateRequest).toBeCalledTimes(2); expect(mockUpdateRequest).toHaveBeenLastCalledWith(requestBFromQueue); @@ -637,7 +643,7 @@ describe('RequestQueue remote', () => { { url: 'http://example.com/6' }, ]); - retrievedUrls.push((await queue.fetchNextRequest()).url); + retrievedUrls.push((await queue.fetchNextRequest())!.url); await queue.addRequest({ url: 'http://example.com/4' }, { forefront: true }); await queue.addRequest({ url: 'http://example.com/3' }, { forefront: true }); @@ -646,12 +652,12 @@ describe('RequestQueue remote', () => { let req = await queue.fetchNextRequest(); - expect(req.url).toBe('http://example.com/2'); + expect(req!.url).toBe('http://example.com/2'); - await queue.reclaimRequest(req, { forefront: true }); + await queue.reclaimRequest(req!, { forefront: true }); while (req) { - retrievedUrls.push(req.url); + retrievedUrls.push(req!.url); req = await queue.fetchNextRequest(); } @@ -711,17 +717,17 @@ describe('RequestQueue remote', () => { method, }); const desc1 = Object.getOwnPropertyDescriptor(r1.userData, '__crawlee'); - expect(desc1.enumerable).toBe(false); + expect(desc1!.enumerable).toBe(false); expect(r1.skipNavigation).toBe(true); expect(r1.maxRetries).toBe(10); r1.maxRetries = 5; expect(r1.userData.__crawlee).toMatchObject({ skipNavigation: true, maxRetries: 5, foo: 123, bar: true }); const desc2 = Object.getOwnPropertyDescriptor(r2.userData, '__crawlee'); - expect(desc2.enumerable).toBe(false); + expect(desc2!.enumerable).toBe(false); expect(r2.maxRetries).toBeUndefined(); expect(r2.userData.__crawlee).toEqual({}); const desc3 = Object.getOwnPropertyDescriptor(r3.userData, '__crawlee'); - expect(desc3.enumerable).toBe(false); + expect(desc3!.enumerable).toBe(false); expect(r3.maxRetries).toBeUndefined(); expect(r3.userData.__crawlee).toEqual({}); r3.maxRetries = 2; @@ -951,7 +957,7 @@ describe('RequestQueue v2', () => { ...Array.from({ length: 25 }, (_, i) => ({ url: `http://example.com/${i + 4}` })), ]); - retrievedUrls.push((await queue.fetchNextRequest()).url); + retrievedUrls.push((await queue.fetchNextRequest())!.url); await queue.addRequest({ url: 'http://example.com/3' }, { forefront: true }); await queue.addRequest({ url: 'http://example.com/2' }, { forefront: true }); @@ -959,7 +965,7 @@ describe('RequestQueue v2', () => { let req = await queue.fetchNextRequest(); while (req) { - retrievedUrls.push(req.url); + retrievedUrls.push(req!.url); req = await queue.fetchNextRequest(); } @@ -980,21 +986,21 @@ describe('RequestQueue v2', () => { { url: 'http://example.com/5' }, ]); - retrievedUrls.push((await queue.fetchNextRequest()).url); + retrievedUrls.push((await queue.fetchNextRequest())!.url); await queue.addRequest({ url: 'http://example.com/3' }, { forefront: true }); await queue.addRequest({ url: 'http://example.com/2' }, { forefront: true }); let req = await queue.fetchNextRequest(); - expect(req.url).toBe('http://example.com/2'); + expect(req!.url).toBe('http://example.com/2'); - await queue.reclaimRequest(req, { forefront: true }); + await queue.reclaimRequest(req!, { forefront: true }); req = await queue.fetchNextRequest(); while (req) { - retrievedUrls.push(req.url); + retrievedUrls.push(req!.url); req = await queue.fetchNextRequest(); } diff --git a/test/core/storages/utils.test.ts b/test/core/storages/utils.test.ts index a103098af819..1ff627c7463f 100644 --- a/test/core/storages/utils.test.ts +++ b/test/core/storages/utils.test.ts @@ -45,7 +45,7 @@ describe('useState', () => { state.hello = 'foo'; state.foo = ['fizz']; - const manager = Configuration.globalConfig.getEventManager(); + const manager = Configuration.getEventManager(); await manager.init(); diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts index af0d6da9e9f1..2019d4503d76 100644 --- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts +++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts @@ -1,6 +1,6 @@ import { Readable } from 'stream'; -import { CheerioCrawler } from '@crawlee/cheerio'; +import { CheerioCrawler, Dictionary } from '@crawlee/cheerio'; import { BaseHttpClient, BaseHttpResponseData, @@ -173,7 +173,7 @@ const crawler = new CheerioCrawler({ responseType: 'json', }); - const { body: ua } = await context.sendRequest({ + const { body: ua } = await context.sendRequest({ url: 'https://httpbin.org/user-agent', responseType: 'json', }); diff --git a/test/shared/MemoryStorageEmulator.ts b/test/shared/MemoryStorageEmulator.ts index 0f2b69b16a67..186396a0d42b 100644 --- a/test/shared/MemoryStorageEmulator.ts +++ b/test/shared/MemoryStorageEmulator.ts @@ -11,7 +11,7 @@ import { StorageEmulator } from './StorageEmulator'; const LOCAL_EMULATION_DIR = resolve(__dirname, '..', 'tmp', 'memory-emulation-dir'); export class MemoryStorageEmulator extends StorageEmulator { - private storage: MemoryStorage; + private storage!: MemoryStorage; override async init({ dirName = cryptoRandomObjectId(10), persistStorage = false }: MemoryEmulatorOptions = {}) { await super.init(); diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts index c309465b1fb7..7b5883916ff5 100644 --- a/test/shared/_helper.ts +++ b/test/shared/_helper.ts @@ -206,8 +206,8 @@ console.log('Hello world!'); - -
+
+