diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts index 11feea642deb0..2bf129097be2b 100644 --- a/x-pack/plugins/reporting/common/constants.ts +++ b/x-pack/plugins/reporting/common/constants.ts @@ -66,6 +66,8 @@ export const DEFAULT_VIEWPORT = { }; // Export Type Definitions +export const CSV_SAVED_OBJECT_JOB_TYPE = 'csv_saved_object'; + export const CSV_REPORT_TYPE = 'CSV'; export const CSV_JOB_TYPE = 'csv_searchsource'; diff --git a/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts b/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts new file mode 100644 index 0000000000000..cddac3aab8218 --- /dev/null +++ b/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BaseParams, BasePayload } from '../base'; + +interface CsvFromSavedObjectBase { + objectType: 'saved search'; + timerange?: { + timezone?: string; + min?: string | number; + max?: string | number; + }; + savedObjectId: string; +} + +export type JobParamsCsvFromSavedObject = CsvFromSavedObjectBase & BaseParams; +export type TaskPayloadCsvFromSavedObject = CsvFromSavedObjectBase & BasePayload; diff --git a/x-pack/plugins/reporting/common/types/export_types/index.ts b/x-pack/plugins/reporting/common/types/export_types/index.ts index 0ad47e7414031..eb9242c76e392 100644 --- a/x-pack/plugins/reporting/common/types/export_types/index.ts +++ b/x-pack/plugins/reporting/common/types/export_types/index.ts @@ -8,6 +8,7 @@ export * from './csv'; export * from './csv_searchsource'; export * from './csv_searchsource_immediate'; +export * from './csv_saved_object'; export * from './png'; export * from './png_v2'; export * from './printable_pdf'; diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index 533f8da2a98c5..22c2b1747fc60 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -259,7 +259,7 @@ export class ReportingCore { return this.pluginSetupDeps; } - private async getSavedObjectsClient(request: KibanaRequest) { + public async getSavedObjectsClient(request: KibanaRequest) { const { savedObjects } = await this.getPluginStartDeps(); return savedObjects.getScopedClient(request) as SavedObjectsClientContract; } diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/create_job.ts new file mode 100644 index 0000000000000..6751dea11018e --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/create_job.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CreateJobFn, CreateJobFnFactory } from '../../types'; +import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from './types'; + +type CreateJobFnType = CreateJobFn; + +export const createJobFnFactory: CreateJobFnFactory = + function createJobFactoryFn() { + return async function createJob(jobParams) { + // params have been finalized in server/routes/generate_from_savedobject.ts + return jobParams; + }; + }; diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.test.ts new file mode 100644 index 0000000000000..5f8cba7e52f4a --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.test.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +jest.mock('../csv_searchsource/generate_csv', () => ({ + CsvGenerator: class CsvGeneratorMock { + generateData() { + return { + size: 123, + content_type: 'text/csv', + }; + } + }, +})); + +jest.mock('./lib/get_sharing_data', () => ({ + getSharingData: jest.fn(() => ({ columns: [], searchSource: {} })), +})); + +import { Writable } from 'stream'; +import nodeCrypto from '@elastic/node-crypto'; +import { ReportingCore } from '../../'; +import { CancellationToken } from '../../../common'; +import { + createMockConfigSchema, + createMockLevelLogger, + createMockReportingCore, +} from '../../test_helpers'; +import { runTaskFnFactory } from './execute_job'; + +const logger = createMockLevelLogger(); +const encryptionKey = 'tetkey'; +const headers = { sid: 'cooltestheaders' }; +let encryptedHeaders: string; +let reportingCore: ReportingCore; +let stream: jest.Mocked; + +beforeAll(async () => { + const crypto = nodeCrypto({ encryptionKey }); + encryptedHeaders = await crypto.encrypt(headers); +}); + +beforeEach(async () => { + stream = {} as typeof stream; + reportingCore = await createMockReportingCore(createMockConfigSchema({ encryptionKey })); +}); + +test('recognized saved search', async () => { + reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({ + get: () => ({ + attributes: { + kibanaSavedObjectMeta: { + searchSourceJSON: '{"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, + }, + references: [ + { + id: 'logstash-yes-*', + name: 'kibanaSavedObjectMeta.searchSourceJSON.index', + type: 'index-pattern', + }, + ], + }), + }); + + const runTask = runTaskFnFactory(reportingCore, logger); + const payload = await runTask( + 'cool-job-id', + { + headers: encryptedHeaders, + browserTimezone: 'US/Alaska', + savedObjectId: '123-456-abc-defgh', + objectType: 'saved search', + title: 'Test Search', + version: '7.17.0', + }, + new CancellationToken(), + stream + ); + + expect(payload).toMatchInlineSnapshot(` + Object { + "content_type": "text/csv", + "size": 123, + } + `); +}); + +test('saved search object is missing references', async () => { + reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({ + get: () => ({ + attributes: { + kibanaSavedObjectMeta: { + searchSourceJSON: '{"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, + }, + }), + }); + + const runTask = runTaskFnFactory(reportingCore, logger); + const runTest = async () => { + await runTask( + 'cool-job-id', + { + headers: encryptedHeaders, + browserTimezone: 'US/Alaska', + savedObjectId: '123-456-abc-defgh', + objectType: 'saved search', + title: 'Test Search', + version: '7.17.0', + }, + new CancellationToken(), + stream + ); + }; + + await expect(runTest).rejects.toEqual( + new Error('Could not find reference for kibanaSavedObjectMeta.searchSourceJSON.index') + ); +}); + +test('invalid saved search', async () => { + reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({ get: jest.fn() }); + const runTask = runTaskFnFactory(reportingCore, logger); + const runTest = async () => { + await runTask( + 'cool-job-id', + { + headers: encryptedHeaders, + browserTimezone: 'US/Alaska', + savedObjectId: '123-456-abc-defgh', + objectType: 'saved search', + title: 'Test Search', + version: '7.17.0', + }, + new CancellationToken(), + stream + ); + }; + + await expect(runTest).rejects.toEqual(new Error('Saved search object is not valid')); +}); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts new file mode 100644 index 0000000000000..e748d847b86f0 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from 'kibana/server'; +import type { SearchSourceFields } from 'src/plugins/data/common'; +import type { VisualizationSavedObjectAttributes } from 'src/plugins/visualizations/common'; +import { DeepPartial } from 'utility-types'; +import { JobParamsCSV } from '../..'; +import { injectReferences, parseSearchSourceJSON } from '../../../../../../src/plugins/data/common'; +import { CSV_JOB_TYPE } from '../../../common/constants'; +import { getFieldFormats } from '../../services'; +import type { RunTaskFn, RunTaskFnFactory } from '../../types'; +import { decryptJobHeaders } from '../common'; +import { CsvGenerator } from '../csv_searchsource/generate_csv'; +import { getSharingData } from './lib'; +import type { TaskPayloadCsvFromSavedObject } from './types'; + +type RunTaskFnType = RunTaskFn; +type SavedSearchObjectType = SavedObject< + VisualizationSavedObjectAttributes & { columns?: string[]; sort: Array<[string, string]> } +>; +type ParsedSearchSourceJSON = SearchSourceFields & { indexRefName?: string }; + +function isSavedObject( + savedSearch: SavedSearchObjectType | unknown +): savedSearch is SavedSearchObjectType { + return ( + (savedSearch as DeepPartial | undefined)?.attributes + ?.kibanaSavedObjectMeta?.searchSourceJSON != null + ); +} + +export const runTaskFnFactory: RunTaskFnFactory = (reporting, _logger) => { + const config = reporting.getConfig(); + + return async function runTask(jobId, job, cancellationToken, stream) { + const logger = _logger.clone([CSV_JOB_TYPE, 'execute-job', jobId]); + + const encryptionKey = config.get('encryptionKey'); + const headers = await decryptJobHeaders(encryptionKey, job.headers, logger); + const fakeRequest = reporting.getFakeRequest({ headers }, job.spaceId, logger); + const uiSettings = await reporting.getUiSettingsClient(fakeRequest, logger); + const savedObjects = await reporting.getSavedObjectsClient(fakeRequest); + const dataPluginStart = await reporting.getDataService(); + const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(uiSettings); + + const [es, searchSourceStart] = await Promise.all([ + (await reporting.getEsClient()).asScoped(fakeRequest), + await dataPluginStart.search.searchSource.asScoped(fakeRequest), + ]); + + const clients = { + uiSettings, + data: dataPluginStart.search.asScoped(fakeRequest), + es, + }; + const dependencies = { + searchSourceStart, + fieldFormatsRegistry, + }; + + // Get the Saved Search Fields object from ID + const savedSearch = await savedObjects.get('search', job.savedObjectId); + + if (!isSavedObject(savedSearch)) { + throw new Error(`Saved search object is not valid`); + } + + // allowed to throw an Invalid JSON error if the JSON is not parseable. + const searchSourceFields: ParsedSearchSourceJSON = parseSearchSourceJSON( + savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON + ); + + const indexRefName = searchSourceFields.indexRefName; + if (!indexRefName) { + throw new Error(`Saved Search data is missing a reference to an Index Pattern!`); + } + + // Inject references into the Saved Search Fields + const searchSourceFieldsWithRefs = injectReferences( + { ...searchSourceFields, indexRefName }, + savedSearch.references ?? [] + ); + + // Form the Saved Search attributes and SearchSource into a config that's compatible with CsvGenerator + const { columns, searchSource } = await getSharingData( + { uiSettings }, + await searchSourceStart.create(searchSourceFieldsWithRefs), + savedSearch, + job.timerange + ); + + const jobParamsCsv: JobParamsCSV = { ...job, columns, searchSource }; + const csv = new CsvGenerator( + jobParamsCsv, + config, + clients, + dependencies, + cancellationToken, + logger, + stream + ); + return await csv.generateData(); + }; +}; diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/index.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/index.ts new file mode 100644 index 0000000000000..be29b1e870fc8 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/index.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CSV_SAVED_OBJECT_JOB_TYPE as CSV_JOB_TYPE, + LICENSE_TYPE_BASIC, + LICENSE_TYPE_ENTERPRISE, + LICENSE_TYPE_GOLD, + LICENSE_TYPE_PLATINUM, + LICENSE_TYPE_CLOUD_STANDARD, + LICENSE_TYPE_TRIAL, +} from '../../../common/constants'; +import { CreateJobFn, ExportTypeDefinition, RunTaskFn } from '../../types'; +import { createJobFnFactory } from './create_job'; +import { runTaskFnFactory } from './execute_job'; +import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from './types'; + +export const getExportType = (): ExportTypeDefinition< + CreateJobFn, + RunTaskFn +> => ({ + id: CSV_JOB_TYPE, + name: CSV_JOB_TYPE, + jobType: CSV_JOB_TYPE, + jobContentExtension: 'csv', + createJobFnFactory, + runTaskFnFactory, + validLicenses: [ + LICENSE_TYPE_TRIAL, + LICENSE_TYPE_BASIC, + LICENSE_TYPE_CLOUD_STANDARD, + LICENSE_TYPE_GOLD, + LICENSE_TYPE_PLATINUM, + LICENSE_TYPE_ENTERPRISE, + ], +}); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts new file mode 100644 index 0000000000000..0b567fcb80cc6 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock, httpServerMock } from 'src/core/server/mocks'; +import type { DataView, SearchSource } from 'src/plugins/data/common'; +import { stubIndexPattern, stubIndexPatternWithoutTimeField } from 'src/plugins/data/common/stubs'; +import { + DOC_HIDE_TIME_COLUMN_SETTING, + SEARCH_FIELDS_FROM_SOURCE, +} from 'src/plugins/discover/common'; +import type { VisualizationSavedObjectAttributes } from 'src/plugins/visualizations/common'; +import type { SavedSearchObjectType } from './get_sharing_data'; +import { getSharingData } from './get_sharing_data'; + +const createMockSearchSource = () => + ({ + createCopy: jest.fn().mockReturnThis(), + setField: jest.fn().mockReturnThis(), + removeField: jest.fn().mockReturnThis(), + getSerializedFields: jest.fn(), + } as unknown as SearchSource); +const createMockIndexPattern = () => stubIndexPattern; +const createMockIndexPatternWithoutTimeField = () => stubIndexPatternWithoutTimeField; + +describe('get_sharing_data', () => { + let mockIndexPattern: DataView; + let mockSearchSource: SearchSource; + let mockSavedSearch: SavedSearchObjectType; + const mockSearchSourceGetField = (fieldName: string) => { + if (fieldName === 'index') { + return mockIndexPattern; + } + }; + + const coreStart = coreMock.createStart(); + const request = httpServerMock.createKibanaRequest(); + const soClient = coreStart.savedObjects.getScopedClient(request); + const uiSettings = coreStart.uiSettings.asScopedToClient(soClient); + const fooFn = async (settingName: string) => { + if (settingName === DOC_HIDE_TIME_COLUMN_SETTING || settingName === SEARCH_FIELDS_FROM_SOURCE) { + return false as any; + } + throw new Error('not an expected settingName: ' + settingName); + }; + uiSettings.get = jest.fn(fooFn); + + beforeEach(() => { + mockSavedSearch = { + id: '9747bd90-8581-11ed-97c5-596122858f69', + type: 'search', + namespaces: ['default'], + updated_at: '2022-12-28T23:19:23.982Z', + version: 'WzI1LDFd', + attributes: { + columns: ['clientip', 'extension'], + description: '', + grid: {}, + hideChart: false, + kibanaSavedObjectMeta: { + searchSourceJSON: + '{"highlightAll":true,"version":true,"query":{"query":"","language":"kuery"},"filter":[],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', + }, + sort: [['@timestamp', 'desc']], + title: 'A Saved Search', + } as unknown as VisualizationSavedObjectAttributes & { + columns?: string[] | undefined; + sort: Array<[string, string]>; + }, + references: [ + { + id: 'logstash-*', + name: 'kibanaSavedObjectMeta.searchSourceJSON.index', + type: 'index-pattern', + }, + ], + migrationVersion: { search: '7.9.3' }, + coreMigrationVersion: '7.17.9', + }; + mockSearchSource = createMockSearchSource(); + mockSearchSource.getField = jest.fn(mockSearchSourceGetField); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('with search source using columns with automatic time field', async () => { + mockIndexPattern = createMockIndexPattern(); + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "@timestamp", + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(2); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + '@timestamp', + 'clientip', + 'extension', + ]); + }); + + it('with search source with automatic time field and with no columns', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSavedSearch.attributes.columns = []; + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toBe(undefined); + expect(mockSearchSource.setField).toBeCalledTimes(2); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', ['*']); + }); + + it('with saved search containing ["_source"]', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSavedSearch.attributes.columns = ['_source']; + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toBe(undefined); + expect(mockSearchSource.setField).toBeCalledTimes(2); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', ['*']); + }); + + it('with search source using columns without time field', async () => { + mockIndexPattern = createMockIndexPatternWithoutTimeField(); + mockSavedSearch.attributes.sort = []; + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(2); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [{ _score: 'desc' }]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + 'clientip', + 'extension', + ]); + }); + + it('with saved search containing filters', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSearchSource.getField = jest.fn((fieldName) => { + if (fieldName === 'filter') { + return [ + { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }, + { query: { match_phrase: { 'extension.raw': 'gif' } } }, + ]; + } + return mockSearchSourceGetField(fieldName); + }); + + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "@timestamp", + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(3); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + '@timestamp', + 'clientip', + 'extension', + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }, + { query: { match_phrase: { 'extension.raw': 'gif' } } }, + ]); + }); + + it('with saved search containing filters to be combined with time range filter and timezone from the request', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSearchSource.getField = jest.fn((fieldName) => { + if (fieldName === 'filter') { + return [ + { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }, + { query: { match_phrase: { 'extension.raw': 'gif' } } }, + ]; + } + return mockSearchSourceGetField(fieldName); + }); + const mockTimeRangeFilterFromRequest = { + min: '2022-12-29T22:22:22.222Z', + max: '2022-12-30T22:22:22.222Z', + timezone: 'US/Alaska', + }; + + const sharingData = await getSharingData( + { uiSettings }, + mockSearchSource, + mockSavedSearch, + mockTimeRangeFilterFromRequest + ); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "@timestamp", + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(3); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + '@timestamp', + 'clientip', + 'extension', + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { + meta: { index: 'logstash-*' }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2022-12-29T22:22:22.222Z', + lte: '2022-12-30T22:22:22.222Z', + }, + }, + }, + }, + { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }, + { query: { match_phrase: { 'extension.raw': 'gif' } } }, + ]); + }); + + it('with saved search containing filters to be combined with time range filter from the request in epoch_millis', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSearchSource.getField = jest.fn((fieldName) => { + if (fieldName === 'filter') { + return [{ query: { match_phrase: { 'extension.raw': 'gif' } } }]; + } + return mockSearchSourceGetField(fieldName); + }); + const mockTimeRangeFilterFromRequest = { + min: 1671352518736, + max: 1672352518736, + timezone: 'US/Alaska', + }; + + const sharingData = await getSharingData( + { uiSettings }, + mockSearchSource, + mockSavedSearch, + mockTimeRangeFilterFromRequest + ); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "@timestamp", + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(3); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + '@timestamp', + 'clientip', + 'extension', + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { + meta: { index: 'logstash-*' }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2022-12-18T08:35:18.736Z', + lte: '2022-12-29T22:21:58.736Z', + }, + }, + }, + }, + { query: { match_phrase: { 'extension.raw': 'gif' } } }, + ]); + }); +}); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts new file mode 100644 index 0000000000000..979c43b150fa1 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Filter } from '@kbn/es-query'; +import type { IUiSettingsClient, SavedObject } from 'kibana/server'; +import moment from 'moment-timezone'; +import type { ISearchSource } from 'src/plugins/data/common'; +import type { VisualizationSavedObjectAttributes } from 'src/plugins/visualizations/common'; +import { + DOC_HIDE_TIME_COLUMN_SETTING, + SEARCH_FIELDS_FROM_SOURCE, +} from '../../../../../../../src/plugins/discover/common'; +import { getSortForSearchSource } from './get_sort_for_search_source'; + +export type SavedSearchObjectType = SavedObject< + VisualizationSavedObjectAttributes & { columns?: string[]; sort: Array<[string, string]> } +>; + +function isStringArray(arr: unknown | string[]): arr is string[] { + return Array.isArray(arr) && arr.every((p) => typeof p === 'string'); +} + +/** + * Partially copied from src/plugins/discover/public/application/apps/main/utils/get_sharing_data.ts + */ + +export async function getSharingData( + services: { uiSettings: IUiSettingsClient }, + currentSearchSource: ISearchSource, + savedSearch: SavedSearchObjectType, + jobParamsTimeRange?: { min?: string | number; max?: string | number; timezone?: string } +) { + const searchSource = currentSearchSource.createCopy(); + const index = searchSource.getField('index'); + + if (!index) { + throw new Error(`Search Source is missing the "index" field`); + } + + // Inject sort + searchSource.setField('sort', getSortForSearchSource(savedSearch.attributes.sort, index)); + + // Remove the fields that are not suitable for export and paging + searchSource.removeField('highlight'); + searchSource.removeField('highlightAll'); + searchSource.removeField('aggs'); + searchSource.removeField('size'); + + const [hideTimeColumn, useFieldsFromSource] = await Promise.all([ + services.uiSettings.get(DOC_HIDE_TIME_COLUMN_SETTING), + services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE), + ]); + + // Add/adjust columns from the saved search attributes and UI Settings + let columns: string[] | undefined; + let timeFieldName: string | undefined; + // ignore '_source' column: it may be the only column when the user wishes to export all fields + const columnsTemp = savedSearch.attributes?.columns?.filter((col) => col !== '_source'); + + if (typeof columnsTemp !== 'undefined' && columnsTemp.length > 0 && isStringArray(columnsTemp)) { + columns = columnsTemp; + + // conditionally add the time field column: + if (index?.timeFieldName && !hideTimeColumn) { + timeFieldName = index.timeFieldName; + } + if (timeFieldName && !columnsTemp.includes(timeFieldName)) { + columns = [timeFieldName, ...columns]; + } + + /* + * For querying performance, the searchSource object must have fields set. + * Otherwise, the requests will ask for all fields, even if only a few are really needed. + * Discover does not set fields, since having all fields is needed for the UI. + */ + if (!useFieldsFromSource && columns.length) { + searchSource.setField('fields', columns); + } + } else { + searchSource.setField('fields', ['*']); + } + + // Combine the job's time filter into the SearchSource instance + let jobParamsTimeRangeFilter: Filter | undefined; + if ((jobParamsTimeRange?.min || jobParamsTimeRange?.max) && timeFieldName) { + const { min, max } = jobParamsTimeRange; + const timezone = jobParamsTimeRange.timezone ?? 'UTC'; + const minTime = min ? moment.tz(min, timezone) : undefined; + const maxTime = max ? moment.tz(max, timezone) : undefined; + jobParamsTimeRangeFilter = { + meta: { index: index.id }, + query: { + range: { + [timeFieldName]: { + format: 'strict_date_optional_time', + gte: minTime?.toISOString(), + lte: maxTime?.toISOString(), + }, + }, + }, + }; + } + + // Combine the time range filter from the job request body with any filters that have been saved into the saved search object + // NOTE: if the filters that were saved into the search are NOT an array, it may be a function. Function + // filters are not supported in this API. + const savedSearchFilterTmp = searchSource.getField('filter'); + searchSource.removeField('filter'); + + let combinedFilters: Filter[] | undefined; + let savedSearchFilter: Filter[] | undefined; + if (savedSearchFilterTmp && Array.isArray(savedSearchFilterTmp)) { + // can not include functions: could be recursive + savedSearchFilter = [...savedSearchFilterTmp.filter((f) => typeof f !== 'function')]; + } else if (savedSearchFilterTmp && typeof savedSearchFilterTmp !== 'function') { + savedSearchFilter = [savedSearchFilterTmp]; + } + + if (savedSearchFilter && jobParamsTimeRangeFilter) { + combinedFilters = [jobParamsTimeRangeFilter, ...savedSearchFilter]; + } else if (savedSearchFilter) { + combinedFilters = [...savedSearchFilter]; + } else if (jobParamsTimeRangeFilter) { + combinedFilters = [jobParamsTimeRangeFilter]; + } + + if (combinedFilters) { + searchSource.setField('filter', combinedFilters); + } + + return { + columns, + searchSource: searchSource.getSerializedFields(true), + }; +} diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.test.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.test.ts new file mode 100644 index 0000000000000..44abf1108cbdb --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataViewField } from 'src/plugins/data/common'; +import { stubIndexPattern, stubIndexPatternWithoutTimeField } from 'src/plugins/data/common/stubs'; +import { getSort } from './get_sort'; + +const createMockIndexPattern = () => stubIndexPattern; +const createMockIndexPatternWithoutTimeField = () => stubIndexPatternWithoutTimeField; + +describe('get_sort', () => { + it('gets the sort for @timestamp', () => { + const mockIndexPattern = createMockIndexPattern(); + mockIndexPattern.getFieldByName = jest.fn((fieldName) => { + if (fieldName === '@timestamp') { + return { sortable: true } as DataViewField; + } + }); + const sortPair: [string, string] = ['sortField', 'desc']; + + const result = getSort(sortPair, mockIndexPattern); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "sortField": "desc", + }, + ] + `); + }); + + it('gets complex sort', () => { + const mockIndexPattern = createMockIndexPatternWithoutTimeField(); + mockIndexPattern.getFieldByName = jest.fn(() => { + return { + sortable: true, + } as DataViewField; + }); + const sortPair: Array<[string, string]> = [ + ['eon', 'asc'], + ['epoch', 'asc'], + ['era', 'asc'], + ['period', 'asc'], + ]; + + const result = getSort(sortPair, mockIndexPattern); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "eon": "asc", + }, + Object { + "epoch": "asc", + }, + Object { + "era": "asc", + }, + Object { + "period": "asc", + }, + ] + `); + }); +}); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.ts new file mode 100644 index 0000000000000..5dbbf8575b8b4 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * This file was copied from src/plugins/discover/public/application/apps/main/components/doc_table/lib/get_sort.ts + */ + +import { isPlainObject } from 'lodash'; +import type { IndexPattern } from 'src/plugins/data/common'; + +type SortPairObj = Record; +type SortPairArr = [string, string]; +type SortPair = SortPairArr | SortPairObj; +type SortInput = SortPair | SortPair[]; + +function isSortable(fieldName: string, indexPattern: IndexPattern): boolean { + const field = indexPattern.getFieldByName(fieldName); + return !!(field && field.sortable); +} + +function isLegacySort(sort: SortPair[] | SortPair): sort is SortPair { + return ( + sort.length === 2 && typeof sort[0] === 'string' && (sort[1] === 'desc' || sort[1] === 'asc') + ); +} + +function createSortObject( + sortPair: SortInput, + indexPattern: IndexPattern +): SortPairObj | undefined { + if ( + Array.isArray(sortPair) && + sortPair.length === 2 && + isSortable(String(sortPair[0]), indexPattern) + ) { + const [field, direction] = sortPair as SortPairArr; + return { [field]: direction }; + } else if (isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) { + return sortPair as SortPairObj; + } +} + +/** + * Take a sorting array and make it into an object + * @param {array} sort two dimensional array [[fieldToSort, directionToSort]] + * or an array of objects [{fieldToSort: directionToSort}] + * @param {object} indexPattern used for determining default sort + * @returns Array<{object}> an array of sort objects + */ +export function getSort(sort: SortPair[] | SortPair, indexPattern: IndexPattern): SortPairObj[] { + if (Array.isArray(sort)) { + if (isLegacySort(sort)) { + // To stay compatible with legacy sort, which just supported a single sort field + return [{ [sort[0]]: sort[1] }]; + } + return sort + .map((sortPair: SortPair) => createSortObject(sortPair, indexPattern)) + .filter((sortPairObj) => typeof sortPairObj === 'object') as SortPairObj[]; + } + return []; +} diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.test.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.test.ts new file mode 100644 index 0000000000000..340d6a1a8a6ad --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { stubIndexPattern, stubIndexPatternWithoutTimeField } from 'src/plugins/data/common/stubs'; +import { getSortForSearchSource } from './get_sort_for_search_source'; + +// copied from src/plugins/discover/public/application/apps/main/components/doc_table/lib/get_sort_for_search_source.test.ts +describe('getSortForSearchSource function', function () { + test('should be a function', function () { + expect(typeof getSortForSearchSource === 'function').toBeTruthy(); + }); + + test('should return an object to use for searchSource when columns are given', function () { + const cols: Array<[string, string]> = [['bytes', 'desc']]; + expect(getSortForSearchSource(cols, stubIndexPattern)).toEqual([{ bytes: 'desc' }]); + expect(getSortForSearchSource(cols, stubIndexPattern, 'asc')).toEqual([{ bytes: 'desc' }]); + + expect(getSortForSearchSource(cols, stubIndexPatternWithoutTimeField)).toEqual([ + { bytes: 'desc' }, + ]); + expect(getSortForSearchSource(cols, stubIndexPatternWithoutTimeField, 'asc')).toEqual([ + { bytes: 'desc' }, + ]); + }); + + test('should return an object to use for searchSource when no columns are given', function () { + const cols: Array<[string, string]> = []; + expect(getSortForSearchSource(cols, stubIndexPattern)).toEqual([{ _doc: 'desc' }]); + expect(getSortForSearchSource(cols, stubIndexPattern, 'asc')).toEqual([{ _doc: 'asc' }]); + + expect(getSortForSearchSource(cols, stubIndexPatternWithoutTimeField)).toEqual([ + { _score: 'desc' }, + ]); + expect(getSortForSearchSource(cols, stubIndexPatternWithoutTimeField, 'asc')).toEqual([ + { _score: 'asc' }, + ]); + }); +}); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.ts new file mode 100644 index 0000000000000..8143da9d25b27 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sort_for_search_source.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * This file was copied from + * src/plugins/discover/public/application/apps/main/components/doc_table/lib/get_sort_for_search_source.ts + */ + +import type { EsQuerySortValue, IndexPattern } from 'src/plugins/data/common'; +import { getSort } from './get_sort'; + +export function getSortForSearchSource( + sort?: Array<[string, string]>, + indexPattern?: IndexPattern, + defaultDirection: string = 'desc' +): EsQuerySortValue[] { + if (!sort || !indexPattern || (Array.isArray(sort) && sort.length === 0)) { + if (indexPattern?.timeFieldName) { + // sorting by index order + return [{ _doc: defaultDirection } as EsQuerySortValue]; + } else { + return [{ _score: defaultDirection } as EsQuerySortValue]; + } + } + const { timeFieldName } = indexPattern; + return getSort(sort, indexPattern).map((sortPair: Record) => { + if (indexPattern.isTimeNanosBased() && timeFieldName && sortPair[timeFieldName]) { + return { + [timeFieldName]: { + order: sortPair[timeFieldName], + numeric_type: 'date_nanos', + }, + } as EsQuerySortValue; + } + return sortPair as EsQuerySortValue; + }); +} diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/index.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/index.ts new file mode 100644 index 0000000000000..362dcbe78d584 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getSortForSearchSource } from './get_sort_for_search_source'; +export { getSharingData } from './get_sharing_data'; diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/types.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/types.ts new file mode 100644 index 0000000000000..9c27fbd93bb09 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type { + CsvSavedSearchExportParamsType, + CsvSavedSearchExportBodyType, +} from '../../routes/generate/generate_from_savedobject'; + +export type { + JobParamsCsvFromSavedObject, + TaskPayloadCsvFromSavedObject, +} from '../../../common/types'; diff --git a/x-pack/plugins/reporting/server/lib/export_types_registry.ts b/x-pack/plugins/reporting/server/lib/export_types_registry.ts index 314d50e131565..5185be586e043 100644 --- a/x-pack/plugins/reporting/server/lib/export_types_registry.ts +++ b/x-pack/plugins/reporting/server/lib/export_types_registry.ts @@ -7,7 +7,8 @@ import { isString } from 'lodash'; import { getExportType as getTypeCsvDeprecated } from '../export_types/csv'; -import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_searchsource_immediate'; +import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_saved_object'; +import { getExportType as getTypeCsvFromSavedObjectImmediate } from '../export_types/csv_searchsource_immediate'; import { getExportType as getTypeCsv } from '../export_types/csv_searchsource'; import { getExportType as getTypePng } from '../export_types/png'; import { getExportType as getTypePngV2 } from '../export_types/png_v2'; @@ -90,6 +91,7 @@ export function getExportTypesRegistry(): ExportTypesRegistry { getTypeCsv, getTypeCsvDeprecated, getTypeCsvFromSavedObject, + getTypeCsvFromSavedObjectImmediate, getTypePng, getTypePngV2, getTypePrintablePdf, diff --git a/x-pack/plugins/reporting/server/routes/generate/generation_from_jobparams.test.ts b/x-pack/plugins/reporting/server/routes/generate/generate_from_jobparams.test.ts similarity index 100% rename from x-pack/plugins/reporting/server/routes/generate/generation_from_jobparams.test.ts rename to x-pack/plugins/reporting/server/routes/generate/generate_from_jobparams.test.ts diff --git a/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts b/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts new file mode 100644 index 0000000000000..6d9a9ca81d358 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import assert from 'assert'; +import { SavedObject } from 'kibana/server'; +import moment from 'moment'; +import { wrapError } from '..'; +import { ReportingCore } from '../..'; +import { + API_BASE_URL_V1, + CSV_SAVED_OBJECT_JOB_TYPE as CSV_JOB_TYPE, +} from '../../../common/constants'; +import { JobParamsCsvFromSavedObject } from '../../../common/types'; +import { LevelLogger } from '../../lib'; +import { authorizedUserPreRouting } from '../lib/authorized_user_pre_routing'; +import { RequestHandler } from '../lib/request_handler'; + +const CsvSavedSearchExportParamsSchema = schema.object({ + savedObjectId: schema.string({ minLength: 2 }), +}); + +const CsvSavedSearchExportBodySchema = schema.nullable( + schema.object({ + timerange: schema.maybe( + schema.object({ + timezone: schema.maybe(schema.string()), + min: schema.maybe(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])), + max: schema.maybe(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])), + }) + ), + }) +); + +/** + * Register an API Endpoint for queuing report jobs + * Only CSV export from Saved Search ID is supported. + * @public + */ +export function registerGenerateFromSavedObject(reporting: ReportingCore, logger: LevelLogger) { + const csvSavedSearchExportPath = `${API_BASE_URL_V1}/generate/csv/saved-object/search:{savedObjectId}`; + const setupDeps = reporting.getPluginSetupDeps(); + const { router } = setupDeps; + + router.post( + { + path: csvSavedSearchExportPath, + validate: { + params: CsvSavedSearchExportParamsSchema, + body: CsvSavedSearchExportBodySchema, + }, + }, + authorizedUserPreRouting(reporting, async (user, context, req, res) => { + // 1. Parse the optional time range for validation + let minTime: moment.Moment | undefined; + let maxTime: moment.Moment | undefined; + if (req.body?.timerange?.min || req.body?.timerange?.max) { + try { + minTime = req.body?.timerange?.min ? moment(req.body?.timerange?.min) : minTime; + maxTime = req.body?.timerange?.max ? moment(req.body?.timerange?.max) : maxTime; + if (minTime) assert(minTime.isValid(), `Min time is not valid`); + if (maxTime) assert(maxTime.isValid(), `Max time is not valid`); + } catch (err) { + return res.badRequest(wrapError(err)); + } + } + + try { + // 2. Read the saved object to get the title + const searchObject: SavedObject<{ title?: string }> = + await context.core.savedObjects.client.get('search', req.params.savedObjectId); + + // 3. Store the job params in the Report queue + const requestHandler = new RequestHandler(reporting, user, context, req, res, logger); + + const jobParams: JobParamsCsvFromSavedObject = { + browserTimezone: req.body?.timerange?.timezone || 'UTC', + timerange: req.body?.timerange, + savedObjectId: req.params.savedObjectId, + title: searchObject.attributes.title ?? 'Unknown search', + objectType: 'saved search', + version: '7.17', + }; + + const result = await requestHandler.handleGenerateRequest(CSV_JOB_TYPE, jobParams); + + // 4. Return details of the stored report + return res.ok({ + body: result, + headers: { 'content-type': 'application/json' }, + }); + } catch (err) { + return res.customError(wrapError(err)); + } + }) + ); +} + +export type CsvSavedSearchExportParamsType = TypeOf; +export type CsvSavedSearchExportBodyType = TypeOf; diff --git a/x-pack/plugins/reporting/server/routes/generate/index.ts b/x-pack/plugins/reporting/server/routes/generate/index.ts index 0df9b4a725768..2c7b9793de682 100644 --- a/x-pack/plugins/reporting/server/routes/generate/index.ts +++ b/x-pack/plugins/reporting/server/routes/generate/index.ts @@ -7,4 +7,5 @@ export { registerGenerateCsvFromSavedObjectImmediate } from './csv_searchsource_immediate'; // FIXME: should not need to register each immediate export type separately export { registerJobGenerationRoutes } from './generate_from_jobparams'; +export { registerGenerateFromSavedObject } from './generate_from_savedobject'; export { registerLegacy } from './legacy'; diff --git a/x-pack/plugins/reporting/server/routes/index.ts b/x-pack/plugins/reporting/server/routes/index.ts index 14a16e563ccbb..3f4a355091361 100644 --- a/x-pack/plugins/reporting/server/routes/index.ts +++ b/x-pack/plugins/reporting/server/routes/index.ts @@ -5,12 +5,15 @@ * 2.0. */ +import Boom from '@hapi/boom'; +import { CustomHttpResponseOptions, ResponseError } from 'kibana/server'; import { ReportingCore } from '..'; import { LevelLogger } from '../lib'; import { registerDeprecationsRoutes } from './deprecations'; import { registerDiagnosticRoutes } from './diagnostic'; import { registerGenerateCsvFromSavedObjectImmediate, + registerGenerateFromSavedObject, registerJobGenerationRoutes, registerLegacy, } from './generate'; @@ -20,7 +23,17 @@ export function registerRoutes(reporting: ReportingCore, logger: LevelLogger) { registerDeprecationsRoutes(reporting, logger); registerDiagnosticRoutes(reporting, logger); registerGenerateCsvFromSavedObjectImmediate(reporting, logger); + registerGenerateFromSavedObject(reporting, logger); registerJobGenerationRoutes(reporting, logger); registerLegacy(reporting, logger); registerJobInfoRoutes(reporting); } + +export function wrapError(error: any): CustomHttpResponseOptions { + const boom = Boom.isBoom(error) ? error : Boom.boomify(error, { statusCode: error.statusCode }); + return { + body: boom, + headers: boom.output.headers as { [key: string]: string }, + statusCode: boom.output.statusCode, + }; +} diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 2017ae0be59c7..4580c1c4ffc35 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -191,6 +191,65 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, + "available": Object { + "type": "boolean", + }, + "deprecated": Object { + "type": "long", + }, + "layout": Object { + "canvas": Object { + "type": "long", + }, + "preserve_layout": Object { + "type": "long", + }, + "print": Object { + "type": "long", + }, + }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + "total": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": Object { @@ -493,6 +552,65 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, + "available": Object { + "type": "boolean", + }, + "deprecated": Object { + "type": "long", + }, + "layout": Object { + "canvas": Object { + "type": "long", + }, + "preserve_layout": Object { + "type": "long", + }, + "print": Object { + "type": "long", + }, + }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + "total": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": Object { @@ -813,6 +931,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -913,6 +1045,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1013,6 +1159,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1113,6 +1273,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1213,6 +1387,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1474,6 +1662,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1574,6 +1776,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1674,6 +1890,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1774,6 +2004,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1874,6 +2118,20 @@ Object { "type": "long", }, }, + "csv_saved_object": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "csv_searchsource": Object { "canvas workpad": Object { "type": "long", @@ -1991,6 +2249,22 @@ Object { "output_size": undefined, "total": 4, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2076,6 +2350,22 @@ Object { "output_size": undefined, "total": 4, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2260,6 +2550,22 @@ Object { }, "total": 0, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2343,6 +2649,22 @@ Object { }, "total": 0, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2510,6 +2832,22 @@ Object { "output_size": undefined, "total": 1, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2595,6 +2933,22 @@ Object { "output_size": undefined, "total": 1, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2786,6 +3140,22 @@ Object { "output_size": undefined, "total": 1, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, @@ -2870,6 +3240,22 @@ Object { }, "total": 0, }, + "csv_saved_object": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "layout": Object { + "canvas": 0, + "preserve_layout": 0, + "print": 0, + }, + "total": 0, + }, "csv_searchsource": Object { "app": Object { "canvas workpad": 0, diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index 9580ddb935dfb..afb23b28eb3d4 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -32,6 +32,7 @@ const layoutCountsSchema: MakeSchemaFrom = { const byAppCountsSchema: MakeSchemaFrom = { csv: appCountsSchema, + csv_saved_object: appCountsSchema, csv_searchsource: appCountsSchema, csv_searchsource_immediate: appCountsSchema, PNG: appCountsSchema, @@ -61,6 +62,7 @@ const availableTotalSchema: MakeSchemaFrom = { const jobTypesSchema: MakeSchemaFrom = { csv: availableTotalSchema, + csv_saved_object: availableTotalSchema, csv_searchsource: availableTotalSchema, csv_searchsource_immediate: availableTotalSchema, PNG: availableTotalSchema, diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 856d3ad10cb26..9acd33c4410d0 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -90,6 +90,7 @@ export interface AvailableTotal { // FIXME: find a way to get this from exportTypesHandler or common/constants type BaseJobTypes = | 'csv' + | 'csv_saved_object' | 'csv_searchsource' | 'csv_searchsource_immediate' | 'PNG' diff --git a/x-pack/plugins/reporting/tsconfig.json b/x-pack/plugins/reporting/tsconfig.json index 3e58450565720..068d4de077723 100644 --- a/x-pack/plugins/reporting/tsconfig.json +++ b/x-pack/plugins/reporting/tsconfig.json @@ -14,12 +14,13 @@ ], "references": [ { "path": "../../../src/core/tsconfig.json" }, - { "path": "../../../src/plugins/data/tsconfig.json"}, + { "path": "../../../src/plugins/data/tsconfig.json" }, { "path": "../../../src/plugins/discover/tsconfig.json" }, { "path": "../../../src/plugins/embeddable/tsconfig.json" }, { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, { "path": "../../../src/plugins/management/tsconfig.json" }, { "path": "../../../src/plugins/screenshot_mode/tsconfig.json" }, + { "path": "../../../src/plugins/visualizations/tsconfig.json" }, { "path": "../../../src/plugins/share/tsconfig.json" }, { "path": "../../../src/plugins/ui_actions/tsconfig.json" }, { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, @@ -27,6 +28,6 @@ { "path": "../features/tsconfig.json" }, { "path": "../licensing/tsconfig.json" }, { "path": "../security/tsconfig.json" }, - { "path": "../spaces/tsconfig.json" }, + { "path": "../spaces/tsconfig.json" } ] } diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 3fe3864d8550b..0efd4ef48e8ae 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -4205,6 +4205,73 @@ } } }, + "csv_saved_object": { + "properties": { + "available": { + "type": "boolean" + }, + "total": { + "type": "long" + }, + "deprecated": { + "type": "long" + }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, + "app": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, + "layout": { + "properties": { + "canvas": { + "type": "long" + }, + "print": { + "type": "long" + }, + "preserve_layout": { + "type": "long" + } + } + } + } + }, "csv_searchsource": { "properties": { "available": { @@ -4649,6 +4716,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -4765,6 +4848,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -4881,6 +4980,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -4997,6 +5112,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -5113,6 +5244,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -5316,6 +5463,73 @@ } } }, + "csv_saved_object": { + "properties": { + "available": { + "type": "boolean" + }, + "total": { + "type": "long" + }, + "deprecated": { + "type": "long" + }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, + "app": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, + "layout": { + "properties": { + "canvas": { + "type": "long" + }, + "print": { + "type": "long" + }, + "preserve_layout": { + "type": "long" + } + } + } + } + }, "csv_searchsource": { "properties": { "available": { @@ -5760,6 +5974,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -5876,6 +6106,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -5992,6 +6238,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -6108,6 +6370,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { @@ -6224,6 +6502,22 @@ } } }, + "csv_saved_object": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "csv_searchsource": { "properties": { "search": { diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json new file mode 100644 index 0000000000000..e2092df5507d5 --- /dev/null +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json @@ -0,0 +1,225 @@ +{ + "attributes": { + "fields": "[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]", + "timeFieldName": "@timestamp", + "title": "logstash-*" + }, + "coreMigrationVersion": "7.17.9", + "id": "logstash-*", + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2022-12-29T17:13:37.050Z", + "version": "WzE4LDFd" +} + +{ + "attributes": { + "columns": [ + "clientip", + "extension" + ], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"extension.raw\",\"params\":{\"query\":\"gif\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"extension.raw\":\"gif\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "A Saved Search with a terms filter" + }, + "coreMigrationVersion": "7.17.9", + "id": "339d7a10-86e3-11ed-a42f-1b290a5e335d", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T17:13:37.050Z", + "version": "WzE5LDFd" +} + +{ + "attributes": { + "columns": [ + "clientip", + "extension" + ], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":\"datefilter🥜\",\"negate\":false,\"type\":\"range\",\"key\":\"@timestamp\",\"params\":{\"gte\":\"2015-09-20T10:19:40.307Z\",\"lt\":\"2015-09-20T10:26:56.221Z\"},\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"$state\":{\"store\":\"appState\"},\"query\":{\"range\":{\"@timestamp\":{\"gte\":\"2015-09-20T10:19:40.307Z\",\"lt\":\"2015-09-20T10:26:56.221Z\"}}}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"extension.raw\",\"params\":{\"query\":\"gif\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"extension.raw\":\"gif\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "A Saved Search with date and terms filters" + }, + "coreMigrationVersion": "7.17.9", + "id": "53193950-8649-11ed-9cfd-b9cddf37f461", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T17:13:37.050Z", + "version": "WzIwLDFd" +} + +{ + "attributes": { + "columns": [ + "clientip", + "extension" + ], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "A Saved Search" + }, + "coreMigrationVersion": "7.17.9", + "id": "9747bd90-8581-11ed-97c5-596122858f69", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T17:13:37.050Z", + "version": "WzIxLDFd" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"params\":{\"lt\":\"2015-09-21T04:32:48.778+00:00\",\"gte\":\"2015-09-21T04:02:04.265+00:00\"},\"field\":\"@timestamp\",\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"range\",\"key\":\"@timestamp\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"range\":{\"@timestamp\":{\"lt\":\"2015-09-21T04:32:48.778+00:00\",\"gte\":\"2015-09-21T04:02:04.265+00:00\"}}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Saved Search with date filter and no columns selected" + }, + "coreMigrationVersion": "7.17.9", + "id": "a5698f80-879c-11ed-b087-77d36820f941", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T17:17:17.822Z", + "version": "Wzc5LDFd" +} + +{ + "attributes": { + "columns": [ + "clientip", + "extension" + ], + "description": "", + "grid": {}, + "hideChart": false, + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":\"datefilter🥜\",\"negate\":false,\"type\":\"range\",\"key\":\"@timestamp\",\"params\":{\"gte\":\"2015-09-20T10:19:40.307Z\",\"lt\":\"2015-09-20T10:26:56.221Z\"},\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"$state\":{\"store\":\"appState\"},\"query\":{\"range\":{\"@timestamp\":{\"gte\":\"2015-09-20T10:19:40.307Z\",\"lt\":\"2015-09-20T10:26:56.221Z\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "A Saved Search with a date filter", + "version": 1 + }, + "coreMigrationVersion": "7.17.9", + "id": "d7a79750-3edd-11e9-99cc-4d80163ee9e7", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T17:13:37.050Z", + "version": "WzIyLDFd" +} \ No newline at end of file diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json new file mode 100644 index 0000000000000..87581525809c9 --- /dev/null +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json @@ -0,0 +1,98 @@ +{ + "attributes": { + "fieldAttrs": "{\"eon\":{\"count\":1},\"epoch\":{\"count\":1},\"era\":{\"count\":1},\"period\":{\"count\":1}}", + "fields": "[]", + "runtimeFieldMap": "{}", + "title": "timeless-test", + "typeMeta": "{}" + }, + "coreMigrationVersion": "7.17.9", + "id": "58835c20-87a9-11ed-9097-518b57bf82ea", + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2022-12-29T21:34:51.992Z", + "version": "WzUxLDFd" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [], + "title": "timeless: no filter, no columns selected" + }, + "coreMigrationVersion": "7.17.9", + "id": "6eec8310-87a9-11ed-9097-518b57bf82ea", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "58835c20-87a9-11ed-9097-518b57bf82ea", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T21:34:24.401Z", + "version": "WzI2LDFd" +} + +{ + "attributes": { + "columns": [ + "eon", + "epoch", + "era", + "period" + ], + "description": "", + "grid": {}, + "hideChart": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "eon", + "asc" + ], + [ + "epoch", + "asc" + ], + [ + "era", + "asc" + ], + [ + "period", + "asc" + ] + ], + "title": "timeless: no filter, no columns selected, custom sorting" + }, + "coreMigrationVersion": "7.17.9", + "id": "b38daaf0-87c0-11ed-9bd8-19b3856630c5", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [ + { + "id": "58835c20-87a9-11ed-9097-518b57bf82ea", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-12-29T21:35:23.426Z", + "version": "WzY4LDFd" +} \ No newline at end of file diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap new file mode 100644 index 0000000000000..6cee3c66abadb --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap @@ -0,0 +1,704 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Reporting APIs CSV Generation from Saved Search ID export from non-timebased data view with plain saved search csv file matches (7.17) 1`] = ` +"_id,_index,_score,_type,eon,epoch,era,period +tvJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Phanerozoic, Pliocene,Cenozoic,Neogene +t_JJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Phanerozoic, Holocene,Cenozoic,Quaternary +uPJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Phanerozoic,-,Mesozoic,Cretaceous +ufJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Phanerozoic,-,Mesozoic,Jurassic +uvJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Phanerozoic,-,Paleozoic,Cambrian +u_JJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Proterozoic,-,Paleozoic,Permian +vPJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Archean,-,-,- +vfJJX4UBvD7uFsw9L2x4,timeless-test,1,_doc,Hadean,-,-,- +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from non-timebased data view with plain saved search csv file matches (8) 1`] = ` +"_id,_index,_score,_type,eon,epoch,era,period +tvJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Phanerozoic, Pliocene,Cenozoic,Neogene +t_JJX4UBvD7uFsw9L2x4,timeless-test,1,-,Phanerozoic, Holocene,Cenozoic,Quaternary +uPJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Phanerozoic,-,Mesozoic,Cretaceous +ufJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Phanerozoic,-,Mesozoic,Jurassic +uvJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Phanerozoic,-,Paleozoic,Cambrian +u_JJX4UBvD7uFsw9L2x4,timeless-test,1,-,Proterozoic,-,Paleozoic,Permian +vPJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Archean,-,-,- +vfJJX4UBvD7uFsw9L2x4,timeless-test,1,-,Hadean,-,-,- +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with "doc_table:hideTimeColumn" = "On" csv file matches 1`] = ` +"clientip,extension +238.171.34.42,jpg +155.34.86.215,jpg +231.224.4.183,jpg +17.191.87.129,jpg +239.190.189.77,jpg +237.192.52.3,css +15.202.168.250,jpg +103.57.26.210,jpg +97.83.96.39,jpg +98.89.73.11,jpg +129.18.83.242,gif +53.64.147.41,png +47.141.35.68,css +57.237.11.219,png +51.147.43.175,jpg +42.72.83.65,jpg +0.209.80.244,jpg +87.66.62.130,jpg +236.90.86.83,jpg +44.138.70.255,jpg +159.230.143.48,jpg +51.105.100.214,jpg +16.148.135.166,jpg +220.101.221.163,css +226.167.54.119,css +27.137.8.156,css +226.40.103.254,jpg +228.177.73.18,jpg +7.179.148.96,css +245.69.63.219,png +216.115.81.216,jpg +97.233.183.250,jpg +74.214.76.90,jpg +77.227.154.93,jpg +142.202.198.155,png +238.210.116.210,jpg +45.138.192.138,jpg +206.136.156.225,jpg +180.65.205.98,png +62.132.195.31,jpg +27.127.76.132,jpg +68.107.5.226,jpg +201.154.233.154,jpg +28.3.30.170,jpg +240.3.115.126,jpg +108.46.176.132,jpg +189.34.180.209,jpg +74.224.77.232,gif +121.98.248.112,jpg +194.223.214.184,jpg +135.233.238.181,jpg +169.137.241.169,png +201.154.233.154,png +239.180.70.74,jpg +55.57.7.61,css +222.224.95.15,jpg +13.38.168.2,jpg +111.12.231.216,jpg +230.249.93.77,jpg +252.185.146.124,css +252.59.37.77,jpg +102.19.248.156,php +179.115.77.46,png +239.143.176.28,gif +0.228.1.71,png +71.241.97.89,png +194.232.128.91,png +229.53.125.87,jpg +213.152.13.69,jpg +63.142.94.147,png +84.205.43.205,jpg +216.126.255.31,jpg +158.222.162.158,jpg +194.30.157.224,jpg +150.7.164.59,jpg +248.239.221.43,jpg +89.53.114.44,gif +17.117.135.38,jpg +223.237.148.2,jpg +157.136.137.24,css +174.196.54.80,png +30.36.151.195,jpg +27.127.76.132,jpg +219.136.31.156,png +120.99.203.64,png +194.4.152.176,png +93.12.132.188,css +173.237.216.175,jpg +217.132.169.207,png +241.45.143.98,php +152.43.63.180,css +142.76.208.132,jpg +31.196.214.182,jpg +174.32.230.63,jpg +16.236.101.225,jpg +72.173.69.119,jpg +78.151.142.25,jpg +213.208.15.12,css +30.92.50.149,css +244.112.251.87,jpg +135.233.238.181,gif +238.153.51.78,jpg +111.12.231.216,jpg +9.59.191.54,css +251.207.101.203,jpg +191.83.76.140,png +62.51.178.50,jpg +39.250.210.253,jpg +167.26.56.145,css +194.4.152.176,jpg +144.31.154.6,jpg +21.123.27.0,png +18.113.253.141,jpg +106.217.103.105,css +153.250.60.205,jpg +111.218.253.140,jpg +102.19.248.156,css +197.150.233.60,css +26.225.101.206,jpg +250.208.48.188,css +79.76.124.169,gif +175.99.184.146,jpg +97.147.119.226,png +220.101.221.163,jpg +36.160.226.203,jpg +171.243.18.67,css +185.148.221.121,jpg +12.255.88.79,jpg +90.0.214.93,css +80.83.92.252,jpg +128.169.152.213,jpg +221.228.246.126,png +224.240.85.118,jpg +233.126.159.144,jpg +165.104.70.8,jpg +11.195.111.241,jpg +125.20.91.73,jpg +23.178.170.24,png +28.46.60.20,png +116.171.234.62,jpg +172.218.31.116,jpg +94.154.194.208,jpg +20.34.86.255,jpg +92.96.51.231,jpg +115.225.12.164,jpg +71.169.104.184,css +130.246.231.47,css +44.138.70.255,css +153.250.60.205,jpg +163.123.136.118,png +39.255.142.177,gif +94.48.29.38,css +252.210.163.45,css +53.201.212.220,jpg +25.39.140.144,jpg +90.35.159.150,css +159.136.76.65,gif +68.189.205.188,jpg +220.101.221.163,css +83.236.80.152,css +128.248.96.80,css +234.58.25.145,jpg +81.186.187.11,jpg +162.134.126.82,jpg +142.17.191.204,jpg +137.247.158.26,jpg +156.231.132.106,jpg +22.7.121.157,css +176.7.244.68,jpg +78.233.202.162,jpg +99.16.191.123,jpg +28.46.60.20,jpg +169.137.241.169,jpg +48.242.153.99,css +15.202.168.250,jpg +185.170.80.142,jpg +53.24.249.29,css +215.182.13.142,jpg +120.101.20.104,jpg +51.147.43.175,jpg +62.51.178.50,gif +250.30.209.2,jpg +111.214.104.239,php +112.34.138.226,jpg +185.144.182.162,css +233.126.159.144,css +180.37.241.184,jpg +165.24.164.93,jpg +192.158.203.248,css +184.101.186.240,jpg +93.63.63.137,jpg +126.87.234.213,jpg +111.218.253.140,jpg +155.157.127.149,jpg +54.172.121.192,jpg +88.33.120.224,png +9.69.255.135,jpg +191.77.18.142,css +200.188.1.153,png +57.119.62.60,jpg +103.108.133.125,jpg +98.116.179.248,jpg +16.148.135.166,jpg +18.106.209.231,css +142.202.198.155,jpg +55.119.119.38,jpg +192.158.203.248,png +170.200.84.215,png +108.183.234.114,css +128.108.188.178,jpg +21.123.27.0,gif +213.165.226.140,jpg +147.244.160.168,jpg +1.59.159.138,jpg +158.179.121.61,php +49.95.6.196,gif +140.233.207.177,jpg +55.100.60.111,css +225.42.74.240,jpg +123.85.77.216,png +216.126.255.31,php +63.212.173.15,jpg +240.207.64.202,php +235.115.221.35,jpg +222.224.95.15,jpg +99.74.96.103,jpg +181.135.248.113,jpg +134.47.243.34,css +33.244.83.137,gif +182.186.221.142,jpg +124.187.220.168,jpg +97.83.96.39,jpg +150.7.164.59,jpg +17.123.55.136,jpg +172.218.31.116,jpg +0.53.251.53,jpg +123.35.55.46,css +54.194.215.29,png +25.156.46.205,jpg +167.165.89.8,jpg +17.58.22.109,jpg +160.151.244.234,png +250.44.130.70,jpg +160.244.231.5,jpg +252.107.74.6,jpg +51.70.170.56,jpg +43.123.164.190,jpg +57.79.108.136,jpg +9.112.81.106,png +21.224.184.36,jpg +92.153.81.10,gif +91.217.157.152,jpg +11.195.111.241,jpg +106.142.212.24,jpg +144.135.123.19,css +162.218.161.231,css +200.227.142.190,gif +119.201.32.254,css +203.118.132.112,css +53.55.251.105,jpg +147.195.26.33,jpg +194.109.145.217,jpg +238.237.64.228,css +41.210.252.157,jpg +159.137.252.66,jpg +21.111.249.239,png +238.237.64.228,png +176.253.222.16,jpg +196.97.127.109,jpg +91.10.173.138,jpg +232.144.75.220,css +197.49.80.127,jpg +62.97.175.115,jpg +115.225.12.164,jpg +251.84.111.185,css +114.113.219.30,png +130.171.208.139,jpg +69.87.49.40,gif +252.199.194.23,jpg +188.123.71.54,jpg +69.142.235.222,jpg +160.243.109.222,jpg +82.40.137.38,jpg +54.172.121.192,png +78.233.202.162,jpg +12.200.161.41,jpg +135.90.39.228,jpg +101.130.216.47,gif +95.128.176.48,jpg +235.190.3.193,png +240.55.195.184,css +32.94.46.226,jpg +199.113.69.162,jpg +127.63.221.223,css +156.231.132.106,jpg +240.87.17.249,jpg +187.33.177.173,jpg +177.194.175.66,jpg +126.112.222.13,jpg +126.191.39.198,jpg +197.49.80.127,jpg +142.202.198.155,jpg +254.75.96.91,gif +229.219.133.141,jpg +135.233.238.181,jpg +153.49.130.46,jpg +60.71.214.79,jpg +137.247.158.26,png +110.148.228.141,jpg +25.140.171.133,php +216.242.201.206,jpg +232.206.227.106,jpg +201.154.233.154,jpg +99.216.169.176,gif +44.116.186.0,css +216.126.255.31,css +69.87.49.40,jpg +201.255.221.0,jpg +221.181.166.105,jpg +239.190.189.77,jpg +227.43.145.144,css +172.96.96.191,css +151.97.64.146,jpg +3.6.163.4,jpg +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with no saved filters and job post params csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-20 10:25:44.979,175.188.44.145,css +2015-09-20 10:25:40.968,89.143.125.181,jpg +2015-09-20 10:25:36.331,231.169.195.137,css +2015-09-20 10:25:34.064,137.205.146.206,jpg +2015-09-20 10:25:32.312,53.0.188.251,jpg +2015-09-20 10:25:27.254,111.214.104.239,jpg +2015-09-20 10:25:22.561,111.46.85.146,jpg +2015-09-20 10:25:06.674,55.100.60.111,jpg +2015-09-20 10:25:05.114,34.197.178.155,jpg +2015-09-20 10:24:55.114,163.123.136.118,jpg +2015-09-20 10:24:54.818,11.195.163.57,jpg +2015-09-20 10:24:53.742,96.222.137.213,png +2015-09-20 10:24:48.798,227.228.214.218,jpg +2015-09-20 10:24:20.223,228.53.110.116,jpg +2015-09-20 10:24:01.794,196.131.253.111,png +2015-09-20 10:23:49.521,125.163.133.47,jpg +2015-09-20 10:23:45.816,148.47.216.255,jpg +2015-09-20 10:23:36.052,51.105.100.214,jpg +2015-09-20 10:23:34.323,41.210.252.157,gif +2015-09-20 10:23:27.213,248.163.75.193,png +2015-09-20 10:23:14.866,48.43.210.167,png +2015-09-20 10:23:10.578,33.95.78.209,css +2015-09-20 10:23:07.001,96.40.73.208,css +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with no saved filters and job post params with custom time zone csv file matches 1`] = ` +"order_date,category,currency,customer_id,order_id,day_of_week_i,products.created_on,sku +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories,EUR,19,716724,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085 +2019-07-11 16:00:00.000,Women's Shoes, Women's Clothing,EUR,45,591503,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0006400064, ZO0150601506 +2019-07-11 16:00:00.000,Women's Clothing,EUR,12,591709,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0638206382, ZO0038800388 +2019-07-11 16:00:00.000,Men's Clothing,EUR,52,590937,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0297602976, ZO0565605656 +2019-07-11 16:00:00.000,Men's Clothing,EUR,29,590976,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0561405614, ZO0281602816 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,41,591636,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0385003850, ZO0408604086 +2019-07-11 16:00:00.000,Men's Shoes,EUR,30,591539,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0505605056, ZO0513605136 +2019-07-11 16:00:00.000,Men's Clothing,EUR,41,591598,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0276702767, ZO0291702917 +2019-07-11 16:00:00.000,Women's Clothing,EUR,44,590927,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0046600466, ZO0050800508 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,48,590970,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0455604556, ZO0680806808 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,46,591299,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0229002290, ZO0674406744 +2019-07-11 16:00:00.000,Men's Clothing,EUR,36,591133,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0529905299, ZO0617006170 +2019-07-11 16:00:00.000,Men's Clothing,EUR,13,591175,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0299402994, ZO0433504335 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,21,591297,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0257502575, ZO0451704517 +2019-07-11 16:00:00.000,Men's Clothing,EUR,14,591149,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0584905849, ZO0578405784 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,27,591754,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0335803358, ZO0325903259 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,42,591803,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0645906459, ZO0324303243 +2019-07-11 16:00:00.000,Women's Clothing,EUR,46,592082,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0034400344, ZO0492904929 +2019-07-11 16:00:00.000,Women's Shoes, Women's Accessories,EUR,27,591283,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0239302393, ZO0198501985 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,4,591148,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0290302903, ZO0513705137 +2019-07-11 16:00:00.000,Men's Accessories, Men's Clothing,EUR,51,591417,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0464504645, ZO0621006210 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,14,591562,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0544305443, ZO0108001080 +2019-07-11 16:00:00.000,Women's Clothing, Women's Accessories,EUR,5,590996,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0638106381, ZO0096900969 +2019-07-11 16:00:00.000,Women's Shoes,EUR,27,591317,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0366203662, ZO0139501395 +2019-07-11 16:00:00.000,Men's Clothing,EUR,38,591362,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0541805418, ZO0594105941 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,30,591411,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0693506935, ZO0532405324 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,38,722629,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0424204242, ZO0403504035, ZO0506705067, ZO0395603956 +2019-07-11 16:00:00.000,Men's Clothing,EUR,16,591041,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0418704187, ZO0557105571 +2019-07-11 16:00:00.000,Women's Clothing,EUR,6,591074,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0268602686, ZO0484704847 +2019-07-11 16:00:00.000,Men's Clothing,EUR,7,591349,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0474804748, ZO0560705607 +2019-07-11 16:00:00.000,Women's Accessories, Women's Clothing,EUR,44,591374,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0206002060, ZO0268302683 +2019-07-11 16:00:00.000,Women's Clothing,EUR,12,591230,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0226902269, ZO0660106601 +2019-07-11 16:00:00.000,Women's Shoes, Women's Clothing,EUR,17,591717,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0248002480, ZO0646706467 +2019-07-11 16:00:00.000,Women's Shoes,EUR,42,591768,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0005800058, ZO0133901339 +2019-07-11 16:00:00.000,Men's Clothing,EUR,21,591810,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0587405874, ZO0590305903 +2019-07-11 16:00:00.000,Women's Accessories, Women's Shoes,EUR,22,592049,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0186201862, ZO0018800188 +2019-07-11 16:00:00.000,Women's Clothing, Women's Accessories,EUR,28,592097,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0265502655, ZO0201102011 +2019-07-11 16:00:00.000,Men's Clothing,EUR,19,590807,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0582105821, ZO0421304213 +2019-07-11 16:00:00.000,Men's Accessories, Men's Clothing,EUR,13,714827,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0314503145, ZO0444804448, ZO0538405384, ZO0630306303 +2019-07-11 16:00:00.000,Women's Shoes,EUR,22,591735,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0248402484, ZO0382603826 +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with no saved filters and no job post params csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-22 23:50:13.253,238.171.34.42,jpg +2015-09-22 23:43:58.175,155.34.86.215,jpg +2015-09-22 23:24:14.970,231.224.4.183,jpg +2015-09-22 23:21:38.312,17.191.87.129,jpg +2015-09-22 22:52:43.834,239.190.189.77,jpg +2015-09-22 22:47:07.981,237.192.52.3,css +2015-09-22 22:46:45.525,15.202.168.250,jpg +2015-09-22 22:29:42.029,103.57.26.210,jpg +2015-09-22 22:23:07.177,97.83.96.39,jpg +2015-09-22 22:20:48.857,98.89.73.11,jpg +2015-09-22 22:11:17.488,129.18.83.242,gif +2015-09-22 22:09:30.819,53.64.147.41,png +2015-09-22 22:01:11.244,47.141.35.68,css +2015-09-22 21:49:36.365,57.237.11.219,png +2015-09-22 21:48:33.421,51.147.43.175,jpg +2015-09-22 21:48:11.977,42.72.83.65,jpg +2015-09-22 21:47:34.138,0.209.80.244,jpg +2015-09-22 21:46:01.933,87.66.62.130,jpg +2015-09-22 21:45:59.966,236.90.86.83,jpg +2015-09-22 21:44:49.105,44.138.70.255,jpg +2015-09-22 21:40:33.574,159.230.143.48,jpg +2015-09-22 21:32:58.944,51.105.100.214,jpg +2015-09-22 21:29:40.042,16.148.135.166,jpg +2015-09-22 21:29:37.272,220.101.221.163,css +2015-09-22 21:20:08.993,226.167.54.119,css +2015-09-22 21:07:30.556,27.137.8.156,css +2015-09-22 21:05:31.686,226.40.103.254,jpg +2015-09-22 21:03:46.267,228.177.73.18,jpg +2015-09-22 21:01:53.470,7.179.148.96,css +2015-09-22 20:59:43.107,245.69.63.219,png +2015-09-22 20:58:28.117,216.115.81.216,jpg +2015-09-22 20:57:31.322,97.233.183.250,jpg +2015-09-22 20:54:33.581,74.214.76.90,jpg +2015-09-22 20:53:49.505,77.227.154.93,jpg +2015-09-22 20:50:20.692,142.202.198.155,png +2015-09-22 20:44:08.950,238.210.116.210,jpg +2015-09-22 20:44:05.521,45.138.192.138,jpg +2015-09-22 20:42:36.070,206.136.156.225,jpg +2015-09-22 20:41:53.463,180.65.205.98,png +2015-09-22 20:41:29.385,62.132.195.31,jpg +2015-09-22 20:40:22.952,27.127.76.132,jpg +2015-09-22 20:39:06.213,68.107.5.226,jpg +2015-09-22 20:38:10.646,201.154.233.154,jpg +2015-09-22 20:37:09.278,28.3.30.170,jpg +2015-09-22 20:35:47.356,240.3.115.126,jpg +2015-09-22 20:33:27.729,108.46.176.132,jpg +2015-09-22 20:32:42.845,189.34.180.209,jpg +2015-09-22 20:30:35.890,74.224.77.232,gif +2015-09-22 20:28:19.188,121.98.248.112,jpg +2015-09-22 20:27:47.226,194.223.214.184,jpg +2015-09-22 20:26:31.997,135.233.238.181,jpg +2015-09-22 20:24:19.796,169.137.241.169,png +2015-09-22 20:22:37.423,201.154.233.154,png +2015-09-22 20:21:39.120,239.180.70.74,jpg +2015-09-22 20:19:46.978,55.57.7.61,css +2015-09-22 20:19:04.659,222.224.95.15,jpg +2015-09-22 20:18:51.536,13.38.168.2,jpg +2015-09-22 20:17:31.329,111.12.231.216,jpg +2015-09-22 20:17:23.395,230.249.93.77,jpg +2015-09-22 20:16:00.611,252.185.146.124,css +2015-09-22 20:15:41.764,252.59.37.77,jpg +2015-09-22 20:15:12.828,102.19.248.156,php +2015-09-22 20:14:57.857,179.115.77.46,png +2015-09-22 20:14:54.351,239.143.176.28,gif +2015-09-22 20:13:42.223,0.228.1.71,png +2015-09-22 20:11:39.532,71.241.97.89,png +2015-09-22 20:10:17.108,194.232.128.91,png +2015-09-22 20:10:06.354,229.53.125.87,jpg +2015-09-22 20:08:38.340,213.152.13.69,jpg +2015-09-22 20:07:46.684,63.142.94.147,png +2015-09-22 20:06:41.893,84.205.43.205,jpg +2015-09-22 20:05:41.053,216.126.255.31,jpg +2015-09-22 20:04:22.445,158.222.162.158,jpg +2015-09-22 20:01:46.685,194.30.157.224,jpg +2015-09-22 20:01:25.828,150.7.164.59,jpg +2015-09-22 20:01:09.463,248.239.221.43,jpg +2015-09-22 20:00:22.799,89.53.114.44,gif +2015-09-22 20:00:04.343,17.117.135.38,jpg +2015-09-22 19:59:59.444,223.237.148.2,jpg +2015-09-22 19:58:35.928,157.136.137.24,css +2015-09-22 19:58:17.100,174.196.54.80,png +2015-09-22 19:55:14.208,30.36.151.195,jpg +2015-09-22 19:53:19.530,27.127.76.132,jpg +2015-09-22 19:48:47.191,219.136.31.156,png +2015-09-22 19:48:27.795,120.99.203.64,png +2015-09-22 19:47:24.192,194.4.152.176,png +2015-09-22 19:46:46.305,93.12.132.188,css +2015-09-22 19:45:58.147,173.237.216.175,jpg +2015-09-22 19:45:32.188,217.132.169.207,png +2015-09-22 19:45:13.813,241.45.143.98,php +2015-09-22 19:43:38.182,152.43.63.180,css +2015-09-22 19:42:23.637,142.76.208.132,jpg +2015-09-22 19:42:20.232,31.196.214.182,jpg +2015-09-22 19:42:12.006,174.32.230.63,jpg +2015-09-22 19:40:17.903,16.236.101.225,jpg +2015-09-22 19:39:51.648,72.173.69.119,jpg +2015-09-22 19:39:30.216,78.151.142.25,jpg +2015-09-22 19:39:19.150,213.208.15.12,css +2015-09-22 19:39:05.190,30.92.50.149,css +2015-09-22 19:38:03.689,244.112.251.87,jpg +2015-09-22 19:38:00.511,135.233.238.181,gif +2015-09-22 19:37:58.973,238.153.51.78,jpg +2015-09-22 19:37:50.552,111.12.231.216,jpg +2015-09-22 19:37:41.141,9.59.191.54,css +2015-09-22 19:37:19.681,251.207.101.203,jpg +2015-09-22 19:36:07.805,191.83.76.140,png +2015-09-22 19:35:58.444,62.51.178.50,jpg +2015-09-22 19:34:38.837,39.250.210.253,jpg +2015-09-22 19:33:16.735,167.26.56.145,css +2015-09-22 19:32:50.098,194.4.152.176,jpg +2015-09-22 19:32:23.731,144.31.154.6,jpg +2015-09-22 19:32:12.796,21.123.27.0,png +2015-09-22 19:32:12.353,18.113.253.141,jpg +2015-09-22 19:32:12.321,106.217.103.105,css +2015-09-22 19:31:49.907,153.250.60.205,jpg +2015-09-22 19:29:51.510,111.218.253.140,jpg +2015-09-22 19:29:47.418,102.19.248.156,css +2015-09-22 19:29:34.542,197.150.233.60,css +2015-09-22 19:29:14.959,26.225.101.206,jpg +2015-09-22 19:28:21.352,250.208.48.188,css +2015-09-22 19:25:02.834,79.76.124.169,gif +2015-09-22 19:24:19.169,175.99.184.146,jpg +2015-09-22 19:06:06.687,97.147.119.226,png +2015-09-22 19:05:23.413,220.101.221.163,jpg +2015-09-22 19:04:00.289,36.160.226.203,jpg +2015-09-22 19:03:44.822,171.243.18.67,css +2015-09-22 19:02:50.703,185.148.221.121,jpg +2015-09-22 19:02:46.169,12.255.88.79,jpg +2015-09-22 19:02:26.558,90.0.214.93,css +2015-09-22 19:01:35.168,80.83.92.252,jpg +2015-09-22 19:00:50.612,128.169.152.213,jpg +2015-09-22 18:59:46.757,221.228.246.126,png +2015-09-22 18:59:42.673,224.240.85.118,jpg +2015-09-22 18:59:21.938,233.126.159.144,jpg +2015-09-22 18:59:03.604,165.104.70.8,jpg +2015-09-22 18:59:00.085,11.195.111.241,jpg +2015-09-22 18:58:57.002,125.20.91.73,jpg +2015-09-22 18:58:45.453,23.178.170.24,png +2015-09-22 18:57:51.377,28.46.60.20,png +2015-09-22 18:56:55.069,116.171.234.62,jpg +2015-09-22 18:56:41.217,172.218.31.116,jpg +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with no selected columns and saved date filter and no job post params csv file matches (7.17) 1`] = ` +"@message,@message.raw,@tags,@tags.raw,@timestamp,_id,_index,_score,_type,agent,agent.raw,bytes,clientip,extension,extension.raw,geo.coordinates,geo.dest,geo.src,geo.srcdest,headings,headings.raw,host,host.raw,index,index.raw,ip,links,links.raw,machine.os,machine.os.raw,machine.ram,machine.ram_range,referer,relatedContent.article:modified_time,relatedContent.article:published_time,relatedContent.article:section,relatedContent.article:section.raw,relatedContent.article:tag,relatedContent.article:tag.raw,relatedContent.og:description,relatedContent.og:description.raw,relatedContent.og:image,relatedContent.og:image.raw,relatedContent.og:image:height,relatedContent.og:image:height.raw,relatedContent.og:image:width,relatedContent.og:image:width.raw,relatedContent.og:site_name,relatedContent.og:site_name.raw,relatedContent.og:title,relatedContent.og:title.raw,relatedContent.og:type,relatedContent.og:type.raw,relatedContent.og:url,relatedContent.og:url.raw,relatedContent.twitter:card,relatedContent.twitter:card.raw,relatedContent.twitter:description,relatedContent.twitter:description.raw,relatedContent.twitter:image,relatedContent.twitter:image.raw,relatedContent.twitter:site,relatedContent.twitter:site.raw,relatedContent.twitter:title,relatedContent.twitter:title.raw,relatedContent.url,relatedContent.url.raw,request,request.raw,response,response.raw,spaces,spaces.raw,type,url,url.raw,utc_time,xss,xss.raw +149.170.135.63 - - [2015-09-21T04:32:47.536Z] \\"GET /uploads/vance-brand.jpg HTTP/1.1\\" 200 9723 \\"-\\" \\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\\",149.170.135.63 - - [2015-09-21T04:32:47.536Z] \\"GET /uploads/vance-brand.jpg HTTP/1.1\\" 200 9723 \\"-\\" \\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\\",success, info,success, info,2015-09-21 04:32:47.536,AU_x3_g3GFA8no6QjkHp,logstash-2015.09.21,-,_doc,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,9,723,149.170.135.63,jpg,jpg,{ + \\"coordinates\\": [ + -93.61685667, + 36.38340333 + ], + \\"type\\": \\"Point\\" +},CN,CN,CN:CN,

george-nelson

, http://twitter.com/success/michael-fincke,

george-nelson

, http://twitter.com/success/michael-fincke,media-for-the-masses.theacademyofperformingartsandscience.org,media-for-the-masses.theacademyofperformingartsandscience.org,logstash-2015.09.21,logstash-2015.09.21,149.170.135.63,tamara-e-jernigan@twitter.com, http://twitter.com/info/daniel-barry, www.www.slate.com,tamara-e-jernigan@twitter.com, http://twitter.com/info/daniel-barry, www.www.slate.com,win 7,win 7,15,032,385,536,{ + \\"lt\\": 15032385599, + \\"gte\\": 15032385536 +},http://www.slate.com/warning/kevin-kregel,2015-01-31 23:20:16.000,2008-02-14 16:03:19.000,Music,Music,-,-,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,723,723,480,480,LA Weekly,LA Weekly,Willie Nelson, Nokia, 2/13,Willie Nelson, Nokia, 2/13,article,article,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,summary,summary,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,@laweekly,@laweekly,Willie Nelson, Nokia, 2/13,Willie Nelson, Nokia, 2/13,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,/uploads/vance-brand.jpg,/uploads/vance-brand.jpg,200,200,this is a thing with lots of spaces wwwwoooooo,this is a thing with lots of spaces wwwwoooooo,apache,https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/vance-brand.jpg,https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/vance-brand.jpg,2015-09-21 04:32:47.536,, +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with no selected columns and saved date filter and no job post params csv file matches (8) 1`] = ` +"@message,@message.raw,@tags,@tags.raw,@timestamp,_id,_index,_score,_type,agent,agent.raw,bytes,clientip,extension,extension.raw,geo.coordinates,geo.dest,geo.src,geo.srcdest,headings,headings.raw,host,host.raw,index,index.raw,ip,links,links.raw,machine.os,machine.os.raw,machine.ram,machine.ram_range,referer,relatedContent.article:modified_time,relatedContent.article:published_time,relatedContent.article:section,relatedContent.article:section.raw,relatedContent.article:tag,relatedContent.article:tag.raw,relatedContent.og:description,relatedContent.og:description.raw,relatedContent.og:image,relatedContent.og:image.raw,relatedContent.og:image:height,relatedContent.og:image:height.raw,relatedContent.og:image:width,relatedContent.og:image:width.raw,relatedContent.og:site_name,relatedContent.og:site_name.raw,relatedContent.og:title,relatedContent.og:title.raw,relatedContent.og:type,relatedContent.og:type.raw,relatedContent.og:url,relatedContent.og:url.raw,relatedContent.twitter:card,relatedContent.twitter:card.raw,relatedContent.twitter:description,relatedContent.twitter:description.raw,relatedContent.twitter:image,relatedContent.twitter:image.raw,relatedContent.twitter:site,relatedContent.twitter:site.raw,relatedContent.twitter:title,relatedContent.twitter:title.raw,relatedContent.url,relatedContent.url.raw,request,request.raw,response,response.raw,spaces,spaces.raw,type,url,url.raw,utc_time,xss,xss.raw +149.170.135.63 - - [2015-09-21T04:32:47.536Z] \\"GET /uploads/vance-brand.jpg HTTP/1.1\\" 200 9723 \\"-\\" \\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\\",149.170.135.63 - - [2015-09-21T04:32:47.536Z] \\"GET /uploads/vance-brand.jpg HTTP/1.1\\" 200 9723 \\"-\\" \\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\\",success, info,success, info,2015-09-21 04:32:47.536,AU_x3_g3GFA8no6QjkHp,logstash-2015.09.21,-,-,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,9,723,149.170.135.63,jpg,jpg,{ + \\"coordinates\\": [ + -93.61685667, + 36.38340333 + ], + \\"type\\": \\"Point\\" +},CN,CN,CN:CN,

george-nelson

, http://twitter.com/success/michael-fincke,

george-nelson

, http://twitter.com/success/michael-fincke,media-for-the-masses.theacademyofperformingartsandscience.org,media-for-the-masses.theacademyofperformingartsandscience.org,logstash-2015.09.21,logstash-2015.09.21,149.170.135.63,tamara-e-jernigan@twitter.com, http://twitter.com/info/daniel-barry, www.www.slate.com,tamara-e-jernigan@twitter.com, http://twitter.com/info/daniel-barry, www.www.slate.com,win 7,win 7,15,032,385,536,{ + \\"lt\\": 15032385599, + \\"gte\\": 15032385536 +},http://www.slate.com/warning/kevin-kregel,2015-01-31 23:20:16.000,2008-02-14 16:03:19.000,Music,Music,-,-,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,723,723,480,480,LA Weekly,LA Weekly,Willie Nelson, Nokia, 2/13,Willie Nelson, Nokia, 2/13,article,article,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,summary,summary,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,Willie Nelson, Nokia Theatre, Feb. 13 Photos by Timothy Norris If Willie Nelson had delivered a show with half as much heart, enthusiasm and class, it s...,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,http://IMAGES1.laweekly.com/imager/willie-nelson-nokia-2-13/u/original/2466986/willienelsontn018.jpg,@laweekly,@laweekly,Willie Nelson, Nokia, 2/13,Willie Nelson, Nokia, 2/13,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,http://www.laweekly.com/music/willie-nelson-nokia-2-13-2404749,/uploads/vance-brand.jpg,/uploads/vance-brand.jpg,200,200,this is a thing with lots of spaces wwwwoooooo,this is a thing with lots of spaces wwwwoooooo,apache,https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/vance-brand.jpg,https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/vance-brand.jpg,2015-09-21 04:32:47.536,, +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with saved date and terms filters and no job post params csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-20 10:23:34.323,41.210.252.157,gif +2015-09-20 10:22:36.107,232.64.207.109,gif +2015-09-20 10:19:43.754,171.243.18.67,gif +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with saved date filter and no job post params csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-20 10:26:48.725,74.214.76.90,jpg +2015-09-20 10:26:48.540,146.86.123.109,jpg +2015-09-20 10:26:48.353,233.126.159.144,jpg +2015-09-20 10:26:45.468,153.139.156.196,png +2015-09-20 10:26:34.063,25.140.171.133,css +2015-09-20 10:26:11.181,239.249.202.59,jpg +2015-09-20 10:26:00.639,95.59.225.31,css +2015-09-20 10:26:00.094,247.174.57.245,jpg +2015-09-20 10:25:55.744,116.126.47.226,css +2015-09-20 10:25:54.701,169.228.188.120,jpg +2015-09-20 10:25:52.360,74.224.77.232,css +2015-09-20 10:25:49.913,97.83.96.39,css +2015-09-20 10:25:44.979,175.188.44.145,css +2015-09-20 10:25:40.968,89.143.125.181,jpg +2015-09-20 10:25:36.331,231.169.195.137,css +2015-09-20 10:25:34.064,137.205.146.206,jpg +2015-09-20 10:25:32.312,53.0.188.251,jpg +2015-09-20 10:25:27.254,111.214.104.239,jpg +2015-09-20 10:25:22.561,111.46.85.146,jpg +2015-09-20 10:25:06.674,55.100.60.111,jpg +2015-09-20 10:25:05.114,34.197.178.155,jpg +2015-09-20 10:24:55.114,163.123.136.118,jpg +2015-09-20 10:24:54.818,11.195.163.57,jpg +2015-09-20 10:24:53.742,96.222.137.213,png +2015-09-20 10:24:48.798,227.228.214.218,jpg +2015-09-20 10:24:20.223,228.53.110.116,jpg +2015-09-20 10:24:01.794,196.131.253.111,png +2015-09-20 10:23:49.521,125.163.133.47,jpg +2015-09-20 10:23:45.816,148.47.216.255,jpg +2015-09-20 10:23:36.052,51.105.100.214,jpg +2015-09-20 10:23:34.323,41.210.252.157,gif +2015-09-20 10:23:27.213,248.163.75.193,png +2015-09-20 10:23:14.866,48.43.210.167,png +2015-09-20 10:23:10.578,33.95.78.209,css +2015-09-20 10:23:07.001,96.40.73.208,css +2015-09-20 10:23:02.876,174.32.230.63,jpg +2015-09-20 10:23:00.019,140.233.207.177,jpg +2015-09-20 10:22:47.447,37.127.124.65,jpg +2015-09-20 10:22:45.803,130.171.208.139,png +2015-09-20 10:22:45.590,39.250.210.253,jpg +2015-09-20 10:22:43.997,248.239.221.43,css +2015-09-20 10:22:36.107,232.64.207.109,gif +2015-09-20 10:22:30.527,24.186.122.118,jpg +2015-09-20 10:22:25.697,23.3.174.206,jpg +2015-09-20 10:22:08.272,185.170.80.142,php +2015-09-20 10:21:40.822,202.22.74.232,png +2015-09-20 10:21:36.210,39.227.27.167,jpg +2015-09-20 10:21:19.154,140.233.207.177,jpg +2015-09-20 10:21:09.852,22.151.97.227,jpg +2015-09-20 10:21:06.079,157.39.25.197,css +2015-09-20 10:21:01.357,37.127.124.65,jpg +2015-09-20 10:20:56.519,23.184.94.58,jpg +2015-09-20 10:20:40.189,80.83.92.252,jpg +2015-09-20 10:20:27.012,66.194.157.171,png +2015-09-20 10:20:24.450,15.191.218.38,jpg +2015-09-20 10:19:45.764,199.113.69.162,jpg +2015-09-20 10:19:43.754,171.243.18.67,gif +2015-09-20 10:19:41.208,126.87.234.213,jpg +2015-09-20 10:19:40.307,78.216.173.242,css +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with saved filters and job post params csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-20 10:25:55.744,116.126.47.226,css +2015-09-20 10:25:54.701,169.228.188.120,jpg +2015-09-20 10:25:52.360,74.224.77.232,css +2015-09-20 10:25:49.913,97.83.96.39,css +2015-09-20 10:25:44.979,175.188.44.145,css +2015-09-20 10:25:40.968,89.143.125.181,jpg +2015-09-20 10:25:36.331,231.169.195.137,css +2015-09-20 10:25:34.064,137.205.146.206,jpg +2015-09-20 10:25:32.312,53.0.188.251,jpg +2015-09-20 10:25:27.254,111.214.104.239,jpg +2015-09-20 10:25:22.561,111.46.85.146,jpg +2015-09-20 10:25:06.674,55.100.60.111,jpg +2015-09-20 10:25:05.114,34.197.178.155,jpg +2015-09-20 10:24:55.114,163.123.136.118,jpg +2015-09-20 10:24:54.818,11.195.163.57,jpg +2015-09-20 10:24:53.742,96.222.137.213,png +2015-09-20 10:24:48.798,227.228.214.218,jpg +2015-09-20 10:24:20.223,228.53.110.116,jpg +2015-09-20 10:24:01.794,196.131.253.111,png +2015-09-20 10:23:49.521,125.163.133.47,jpg +2015-09-20 10:23:45.816,148.47.216.255,jpg +2015-09-20 10:23:36.052,51.105.100.214,jpg +" +`; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts new file mode 100644 index 0000000000000..567409e1ab3c5 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts @@ -0,0 +1,574 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { DeepPartial } from 'utility-types'; +import type { ReportApiJSON } from '../../../../x-pack/plugins/reporting/common/types'; +import type { CsvSavedSearchExportBodyType } from '../../../../x-pack/plugins/reporting/server/export_types/csv_saved_object/types'; +import { FtrProviderContext } from '../ftr_provider_context'; + +const LOGSTASH_DATA_ARCHIVE = 'test/functional/fixtures/es_archiver/logstash_functional'; +const LOGSTASH_SAVED_OBJECTS = 'x-pack/test/functional/fixtures/kbn_archiver/reporting/logs'; +const LOGS_SAVED_SEARCH_ID = '9747bd90-8581-11ed-97c5-596122858f69'; +const LOGS_SAVED_SEARCH_NO_COLUMNS_DATE_FILTER_ID = 'a5698f80-879c-11ed-b087-77d36820f941'; +const LOGS_SAVED_SEARCH_DATE_FILTER_ID = 'd7a79750-3edd-11e9-99cc-4d80163ee9e7'; +const LOGS_SAVED_SEARCH_TERMS_FILTER_ID = '53193950-8649-11ed-9cfd-b9cddf37f461'; +const ECOM_SAVED_SEARCH_ID = '6091ead0-1c6d-11ea-a100-8589bb9d7c6b'; +const TIMELESS_SAVED_OBJECTS = 'x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless'; +const TIMELESS_SAVED_SEARCH_ID = '6eec8310-87a9-11ed-9097-518b57bf82ea'; + +const getMockRequestBody = ( + obj: Partial +): CsvSavedSearchExportBodyType => { + return obj + ? { + ...obj, + timerange: { ...obj?.timerange }, + } + : null; +}; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const es = getService('es'); + const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); + const esArchiver = getService('esArchiver'); + const reportingAPI = getService('reportingAPI'); + const log = getService('log'); + const esVersion = getService('esVersion'); + + const requestCsvFromSavedSearch = async ( + savedSearchId: string, + obj: DeepPartial = {} + ) => { + log.info(`sending request for saved search: ${savedSearchId}`); + const body = getMockRequestBody(obj); + return await supertest + .post(`/api/reporting/v1/generate/csv/saved-object/search:${savedSearchId}`) + .set('kbn-xsrf', 'xxx') + .send(body ?? undefined); + }; + + const cleanupLogstash = async () => { + const logstashIndices = await es.indices.get({ + index: 'logstash-*', + allow_no_indices: true, + expand_wildcards: 'all', + ignore_unavailable: true, + }); + await Promise.all( + Object.keys(logstashIndices.body).map(async (logstashIndex) => { + log.info(`deleting ${logstashIndex}`); + await es.indices.delete({ + index: logstashIndex, + }); + }) + ); + }; + + const timelessIndexName = 'timeless-test'; + const loadTimelessData = async () => { + log.info(`loading test data`); + await es.indices.create({ + index: timelessIndexName, + body: { + settings: { number_of_shards: 1 }, + mappings: { + properties: { + eon: { type: 'keyword' }, + era: { type: 'keyword' }, + period: { type: 'keyword' }, + epoch: { type: 'keyword' }, + }, + }, + }, + }); + await es.bulk({ + refresh: 'wait_for', + body: [ + { index: { _index: timelessIndexName, _id: 'tvJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Cenozoic', period: 'Neogene', epoch: ' Pliocene' }, + { index: { _index: timelessIndexName, _id: 't_JJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Cenozoic', period: 'Quaternary', epoch: ' Holocene' }, + { index: { _index: timelessIndexName, _id: 'uPJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Mesozoic', period: 'Cretaceous' }, + { index: { _index: timelessIndexName, _id: 'ufJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Mesozoic', period: 'Jurassic' }, + { index: { _index: timelessIndexName, _id: 'uvJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Paleozoic', period: 'Cambrian' }, + { index: { _index: timelessIndexName, _id: 'u_JJX4UBvD7uFsw9L2x4' } }, + { eon: 'Proterozoic', era: 'Paleozoic', period: 'Permian' }, + { index: { _index: timelessIndexName, _id: 'vPJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Archean' }, + { index: { _index: timelessIndexName, _id: 'vfJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Hadean' }, + ], + }); + }; + + const itIf7 = esVersion.matchRange('<8') ? it : it.skip; + const itIf8 = esVersion.matchRange('>=8') ? it : it.skip; + + describe('CSV Generation from Saved Search ID', () => { + before(async () => { + // clear any previous UI Settings + await kibanaServer.uiSettings.replace({}); + + // explicitly delete all pre-existing logstash indices, since we have exports with no time filter + log.info(`deleting logstash indices`); + await cleanupLogstash(); + + log.info(`updating Advanced Settings`); + await kibanaServer.uiSettings.update({ + 'csv:quoteValues': false, + 'dateFormat:tz': 'UTC', + dateFormat: 'YYYY-MM-DD HH:mm:ss.SSS', + }); + }); + + after(async () => { + await kibanaServer.uiSettings.replace({}); + }); + + describe('export from timebased data view', () => { + before(async () => { + log.info(`loading archives and fixtures`); + await esArchiver.load(LOGSTASH_DATA_ARCHIVE); + await kibanaServer.importExport.load(LOGSTASH_SAVED_OBJECTS); + }); + + after(async () => { + await esArchiver.unload(LOGSTASH_DATA_ARCHIVE); + await kibanaServer.importExport.unload(LOGSTASH_SAVED_OBJECTS); + }); + + describe('export with no saved filters and no job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch(LOGS_SAVED_SEARCH_ID); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with saved date filter and no job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch( + LOGS_SAVED_SEARCH_DATE_FILTER_ID + ); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search with a date filter.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search with a date filter'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with no selected columns and saved date filter and no job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch( + LOGS_SAVED_SEARCH_NO_COLUMNS_DATE_FILTER_ID + ); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="Saved Search with date filter and no columns selected.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('Saved Search with date filter and no columns selected'); + expect(job.payload.version).equal('7.17'); + }); + + itIf7('csv file matches (7.17)', () => { + expectSnapshot(csvFile).toMatch(); + }); + itIf8('csv file matches (8)', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with saved date and terms filters and no job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch( + LOGS_SAVED_SEARCH_TERMS_FILTER_ID + ); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search with date and terms filters.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search with date and terms filters'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with saved filters and job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch( + LOGS_SAVED_SEARCH_DATE_FILTER_ID, + { + timerange: { + min: '2015-09-20 10:23:36.052', + max: '2015-09-20 10:25:55.744', + }, + } + ); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search with a date filter.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search with a date filter'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with no saved filters and job post params', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch(LOGS_SAVED_SEARCH_ID, { + timerange: { + min: '2015-09-20 10:23:07.001', + max: '2015-09-20 10:25:44.979', + }, + }); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('export with no saved filters and job post params with custom time zone', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + await reportingAPI.initEcommerce(); + const { text, status } = await requestCsvFromSavedSearch(ECOM_SAVED_SEARCH_ID, { + timerange: { + timezone: 'US/Alaska', + min: '2019-07-11 00:00:00.000', + max: '2019-07-12 00:00:00.000', + }, + }); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="Ecommerce Data.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + after(async () => { + await reportingAPI.teardownEcommerce(); + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('Ecommerce Data'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe(`export with "doc_table:hideTimeColumn" = "On"`, () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + // make time column not shown by default + await kibanaServer.uiSettings.update({ + // eslint-disable-next-line @typescript-eslint/naming-convention + 'doc_table:hideTimeColumn': true, + }); + + const { text, status } = await requestCsvFromSavedSearch(LOGS_SAVED_SEARCH_ID); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + after(async () => { + await kibanaServer.uiSettings.unset('doc_table:hideTimeColumn'); + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + + describe('validation', () => { + it('with saved search 404', async () => { + const { body } = await requestCsvFromSavedSearch('gobbledygook', {}); + const expectedBody = { + error: 'Not Found', + message: 'Saved object [search/gobbledygook] not found', + statusCode: 404, + }; + expect(body).to.eql(expectedBody); + }); + + it('with invalid min time range', async () => { + const { body } = await requestCsvFromSavedSearch(LOGS_SAVED_SEARCH_ID, { + timerange: { min: `shmurble` }, + }); + const expectedBody = { + error: 'Bad Request', + message: 'Min time is not valid', + statusCode: 400, + }; + expect(body).to.eql(expectedBody); + }); + + it('with invalid max time range', async () => { + const { body } = await requestCsvFromSavedSearch(LOGS_SAVED_SEARCH_ID, { + timerange: { max: `shmurble` }, + }); + const expectedBody = { + error: 'Bad Request', + message: 'Max time is not valid', + statusCode: 400, + }; + expect(body).to.eql(expectedBody); + }); + }); + }); + + describe('export from non-timebased data view', () => { + before(async () => { + await kibanaServer.importExport.load(TIMELESS_SAVED_OBJECTS); + await loadTimelessData(); + }); + + after(async () => { + await kibanaServer.importExport.unload(TIMELESS_SAVED_OBJECTS); + + log.info(`loading test data`); + await es.indices.delete({ + index: timelessIndexName, + }); + }); + + describe('with plain saved search', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch(TIMELESS_SAVED_SEARCH_ID); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="timeless: no filter, no columns selected.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('timeless: no filter, no columns selected'); + expect(job.payload.version).equal('7.17'); + }); + + itIf7('csv file matches (7.17)', () => { + expectSnapshot(csvFile).toMatch(); + }); + itIf8('csv file matches (8)', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + }); + }); +}; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index fc1092487eb42..d3399e8a57967 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -32,6 +32,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./download_csv_dashboard')); loadTestFile(require.resolve('./generate_csv_discover')); loadTestFile(require.resolve('./generate_csv_discover_deprecated')); + loadTestFile(require.resolve('./csv_saved_search')); loadTestFile(require.resolve('./network_policy')); loadTestFile(require.resolve('./spaces')); loadTestFile(require.resolve('./usage'));