Skip to content

Commit

Permalink
fix(core) remove ts error expectation in dataset tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Erik Funder Carstensen committed Dec 21, 2024
1 parent 7cbd7ba commit 4d41334
Showing 1 changed file with 99 additions and 43 deletions.
142 changes: 99 additions & 43 deletions test/core/storages/dataset.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
import { Dataset, checkAndSerialize, chunkBySize, Configuration, KeyValueStore } from '@crawlee/core';
import {
Dataset,
checkAndSerialize,
chunkBySize,
Configuration,
KeyValueStore,
} from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';

Expand Down Expand Up @@ -36,16 +42,22 @@ describe('dataset', () => {
await dataset.pushData({ foo: 'bar' });

expect(mockPushItems).toBeCalledTimes(1);
expect(mockPushItems).toBeCalledWith(JSON.stringify({ foo: 'bar' }));
expect(mockPushItems).toBeCalledWith(
JSON.stringify({ foo: 'bar' })
);

const mockPushItems2 = pushItemSpy.mockResolvedValueOnce(undefined);

await dataset.pushData([{ foo: 'hotel;' }, { foo: 'restaurant' }]);

expect(mockPushItems2).toBeCalledTimes(2);
expect(mockPushItems2).toBeCalledWith(JSON.stringify([{ foo: 'hotel;' }, { foo: 'restaurant' }]));
expect(mockPushItems2).toBeCalledWith(
JSON.stringify([{ foo: 'hotel;' }, { foo: 'restaurant' }])
);

const mockDelete = vitest.spyOn(dataset.client, 'delete').mockResolvedValueOnce(undefined);
const mockDelete = vitest
.spyOn(dataset.client, 'delete')
.mockResolvedValueOnce(undefined);

await dataset.drop();

Expand All @@ -68,8 +80,14 @@ describe('dataset', () => {
await dataset.pushData([{ foo: half }, { bar: half }]);

expect(mockPushItems).toBeCalledTimes(2);
expect(mockPushItems).toHaveBeenNthCalledWith(1, JSON.stringify([{ foo: half }]));
expect(mockPushItems).toHaveBeenNthCalledWith(2, JSON.stringify([{ bar: half }]));
expect(mockPushItems).toHaveBeenNthCalledWith(
1,
JSON.stringify([{ foo: half }])
);
expect(mockPushItems).toHaveBeenNthCalledWith(
2,
JSON.stringify([{ bar: half }])
);
});

test('should successfully save lots of small data', async () => {
Expand Down Expand Up @@ -98,24 +116,40 @@ describe('dataset', () => {

test('should throw on too large file', async () => {
const full = mockData(MAX_PAYLOAD_SIZE_BYTES);
const dataset = new Dataset({ id: 'some-id', client: storageClient });
const dataset = new Dataset({
id: 'some-id',
client: storageClient,
});
try {
await dataset.pushData({ foo: full });
throw new Error('Should fail!');
} catch (err) {
expect(err).toBeInstanceOf(Error);
expect((err as Error).message).toMatch('Data item is too large');
expect((err as Error).message).toMatch(
'Data item is too large'
);
}
});
test('should throw on too large file in an array', async () => {
const full = mockData(MAX_PAYLOAD_SIZE_BYTES);
const dataset = new Dataset({ id: 'some-id', client: storageClient });
const dataset = new Dataset({
id: 'some-id',
client: storageClient,
});
try {
await dataset.pushData([{ foo: 0 }, { foo: 1 }, { foo: 2 }, { foo: full }, { foo: 4 }]);
await dataset.pushData([
{ foo: 0 },
{ foo: 1 },
{ foo: 2 },
{ foo: full },
{ foo: 4 },
]);
throw new Error('Should fail!');
} catch (err) {
expect(err).toBeInstanceOf(Error);
expect((err as Error).message).toMatch('Data item at index 3 is too large');
expect((err as Error).message).toMatch(
'Data item at index 3 is too large'
);
}
});

Expand Down Expand Up @@ -146,21 +180,28 @@ describe('dataset', () => {

expect(result).toEqual(expected);
let e;
const spy = vitest.spyOn(dataset.client, 'listItems').mockImplementation(() => {
throw new Error('Cannot create a string longer than 0x3fffffe7 characters');
});
const spy = vitest
.spyOn(dataset.client, 'listItems')
.mockImplementation(() => {
throw new Error(
'Cannot create a string longer than 0x3fffffe7 characters'
);
});
try {
await dataset.getData();
} catch (err) {
e = err;
}
expect((e as Error).message).toEqual(
'dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.',
'dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.'
);
});

test('getInfo() should work', async () => {
const dataset = new Dataset({ id: 'some-id', client: storageClient });
const dataset = new Dataset({
id: 'some-id',
client: storageClient,
});

const expected: Awaited<ReturnType<Dataset['getInfo']>> = {
id: 'WkzbQMuFYuamGv3YF',
Expand Down Expand Up @@ -234,10 +275,15 @@ describe('dataset', () => {
},
{
limit: 2,
},
}
);
expect(result).toEqual(undefined);
expect(items).toEqual([{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }, { foo: 'd' }]);
expect(items).toEqual([
{ foo: 'a' },
{ foo: 'b' },
{ foo: 'c' },
{ foo: 'd' },
]);
expect(indexes).toEqual([0, 1, 2, 3]);

restoreAndVerify();
Expand All @@ -252,7 +298,7 @@ describe('dataset', () => {
},
{
limit: 2,
},
}
);

expect(result).toEqual([
Expand All @@ -275,7 +321,7 @@ describe('dataset', () => {
},
{
limit: 2,
},
}
);

expect(result).toEqual([
Expand All @@ -296,13 +342,12 @@ describe('dataset', () => {
item.index = index;
item.bar = 'xxx';

// @ts-expect-error FIXME the inference is broken for `reduce()` method
return memo.concat(item);
},
[],
new Array(),
{
limit: 2,
},
}
);

expect(result).toEqual([
Expand All @@ -323,13 +368,12 @@ describe('dataset', () => {
item.index = index;
item.bar = 'xxx';

// @ts-expect-error FIXME the inference is broken for `reduce()` method
return Promise.resolve(memo.concat(item));
},
[],
new Array(),
{
limit: 2,
},
}
);

expect(result).toEqual([
Expand Down Expand Up @@ -369,16 +413,14 @@ describe('dataset', () => {
const calledForIndexes: number[] = [];

const result = await dataset.reduce(
// @ts-expect-error FIXME the inference is broken for `reduce()` method
async (memo, item, index) => {
calledForIndexes.push(index);
// @ts-expect-error FIXME the inference is broken for `reduce()` method
return Promise.resolve(memo.foo > item.foo ? memo : item);
},
undefined,
{
limit: 2,
},
}
);

expect(mockListItems).toBeCalledTimes(2);
Expand All @@ -391,8 +433,7 @@ describe('dataset', () => {
offset: 2,
});

// @ts-expect-error FIXME the inference is broken for `reduce()` method
expect(result.foo).toBe(5);
expect(result!.foo).toBe(5);
expect(calledForIndexes).toEqual([1, 2, 3]);
});
});
Expand All @@ -405,32 +446,34 @@ describe('dataset', () => {
});
// @ts-expect-error JS-side validation
await expect(dataset.pushData()).rejects.toThrow(
'Expected `data` to be of type `object` but received type `undefined`',
'Expected `data` to be of type `object` but received type `undefined`'
);
// @ts-expect-error JS-side validation
await expect(dataset.pushData('')).rejects.toThrow(
'Expected `data` to be of type `object` but received type `string`',
'Expected `data` to be of type `object` but received type `string`'
);
// @ts-expect-error JS-side validation
await expect(dataset.pushData(123)).rejects.toThrow(
'Expected `data` to be of type `object` but received type `number`',
'Expected `data` to be of type `object` but received type `number`'
);
// @ts-expect-error JS-side validation
await expect(dataset.pushData(true)).rejects.toThrow(
'Expected `data` to be of type `object` but received type `boolean`',
'Expected `data` to be of type `object` but received type `boolean`'
);
// @ts-expect-error JS-side validation
await expect(dataset.pushData(false)).rejects.toThrow(
'Expected `data` to be of type `object` but received type `boolean`',
'Expected `data` to be of type `object` but received type `boolean`'
);
await expect(dataset.pushData(() => {})).rejects.toThrow(
'Data item is not an object. You can push only objects into a dataset.',
'Data item is not an object. You can push only objects into a dataset.'
);

const circularObj = {} as Dictionary;
circularObj.xxx = circularObj;
const jsonErrMsg = 'Converting circular structure to JSON';
await expect(dataset.pushData(circularObj)).rejects.toThrow(jsonErrMsg);
await expect(dataset.pushData(circularObj)).rejects.toThrow(
jsonErrMsg
);
});
});

Expand All @@ -454,7 +497,9 @@ describe('dataset', () => {
const str = 'hello';
expect(() => checkAndSerialize(str, 100)).toThrowError(Error);
expect(() => checkAndSerialize([], 100)).toThrowError(Error);
expect(() => checkAndSerialize([str, str], 100)).toThrowError(Error);
expect(() => checkAndSerialize([str, str], 100)).toThrowError(
Error
);
});
test('chunkBySize', () => {
const obj = { foo: 'bar' };
Expand All @@ -478,9 +523,16 @@ describe('dataset', () => {
// Chunks large items individually
expect(chunkBySize(triple, size)).toEqual(triple);
expect(chunkBySize(triple, size + 1)).toEqual(triple);
expect(chunkBySize(triple, size + 2)).toEqual([chunk, chunk, chunk]);
expect(chunkBySize(triple, size + 2)).toEqual([
chunk,
chunk,
chunk,
]);
// Chunks smaller items together
expect(chunkBySize(triple, 2 * size + 3)).toEqual([`[${json},${json}]`, chunk]);
expect(chunkBySize(triple, 2 * size + 3)).toEqual([
`[${json},${json}]`,
chunk,
]);
expect(chunkBySize([...triple, ...triple], 2 * size + 3)).toEqual([
`[${json},${json}]`,
`[${json},${json}]`,
Expand Down Expand Up @@ -544,15 +596,19 @@ describe('dataset', () => {
await dataset.exportToCSV('HELLO-csv');

const kvData = await KeyValueStore.getValue('HELLO-csv');
expect(kvData).toEqual('hello,foo\nworld 1,bar 1\nworld 2,bar 2\nworld 3,bar 3\n');
expect(kvData).toEqual(
'hello,foo\nworld 1,bar 1\nworld 2,bar 2\nworld 3,bar 3\n'
);
});

it('Should work as a static method for the default dataset', async () => {
await Dataset.pushData(dataToPush);
await Dataset.exportToCSV('TEST-123-123-csv');

const kvData = await KeyValueStore.getValue('TEST-123-123-csv');
expect(kvData).toEqual('hello,foo\nworld 1,bar 1\nworld 2,bar 2\nworld 3,bar 3\n');
expect(kvData).toEqual(
'hello,foo\nworld 1,bar 1\nworld 2,bar 2\nworld 3,bar 3\n'
);
});
});
});
Expand Down

0 comments on commit 4d41334

Please sign in to comment.