diff --git a/tdrive/backend/node/src/core/platform/services/search/adapters/elasticsearch/elastic-open-search-adapter.ts b/tdrive/backend/node/src/core/platform/services/search/adapters/elasticsearch/elastic-open-search-adapter.ts index 2102bc772..d6829b7ed 100644 --- a/tdrive/backend/node/src/core/platform/services/search/adapters/elasticsearch/elastic-open-search-adapter.ts +++ b/tdrive/backend/node/src/core/platform/services/search/adapters/elasticsearch/elastic-open-search-adapter.ts @@ -283,11 +283,12 @@ export default class ESAndOpenSearch extends SearchAdapter implements SearchAdap const esParamsWithScroll = { ...esParams, size: parseInt(options.pagination.limitStr || "100"), - scroll: "1m", + scroll: "10m", }; let esResponse: any; + let nextToken; if (options.pagination.page_token) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore @@ -297,18 +298,23 @@ export default class ESAndOpenSearch extends SearchAdapter implements SearchAdap }, esOptions, ); + //the scroll token is also the same, and we do not get it from response + nextToken = options.pagination.page_token; } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore esResponse = await this.client.search(esParamsWithScroll, esOptions); + nextToken = esResponse.body?._scroll_id || ""; } if (esResponse.statusCode !== 200) { logger.error(`${this.name} - ${JSON.stringify(esResponse.body)}`); } - const nextToken = esResponse.body?._scroll_id || ""; const hits = esResponse.body?.hits?.hits || []; + if (hits.length == 0) { + nextToken = false; + } logger.debug(`${this.name} got response: ${JSON.stringify(esResponse)}`); diff --git a/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts b/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts index fc513d60b..a3843e687 100644 --- a/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts +++ b/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts @@ -200,9 +200,10 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI const { query, sort, project } = buildSearchQuery(entityType, filters, options); + logger.info(`Search query: ${JSON.stringify(query)}`); console.log(query); - let cursor = collection.find({ ...query }).sort(sort); + let cursor = collection.find(query).sort(sort); if (project) { cursor = cursor.project(project); } @@ -230,7 +231,7 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI const nextToken = entities.length === parseInt(options.pagination.limitStr) && - (parseInt(options.pagination.page_token) + 1).toString(10); + (parseInt(options.pagination.page_token || "0") + 1).toString(10); const nextPage: Paginable = new Pagination(nextToken, options.pagination.limitStr || "100"); logger.info(`Found ${entities.length} results on entity ${searchPrefix}${index}`); diff --git a/tdrive/backend/node/src/core/platform/services/search/repository.ts b/tdrive/backend/node/src/core/platform/services/search/repository.ts index 359da83d0..516fcdccd 100644 --- a/tdrive/backend/node/src/core/platform/services/search/repository.ts +++ b/tdrive/backend/node/src/core/platform/services/search/repository.ts @@ -40,7 +40,7 @@ export default class SearchRepository { options, ); - //2. Get database original objects from theses primary keys + //2. Get database original objects from these primary keys for (const searchEntity of searchResults.getEntities().sort((a, b) => b.score - a.score)) { const sourceEntity = await repository.findOne(searchEntity.primaryKey, {}, context); if (sourceEntity) { diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 9a90cac41..b2fa2bc9f 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -1,6 +1,10 @@ import SearchRepository from "../../../core/platform/services/search/repository"; import { getLogger, logger, TdriveLogger } from "../../../core/platform/framework"; -import { CrudException, ListResult } from "../../../core/platform/framework/api/crud-service"; +import { + CrudException, + ListResult, + Pagination, +} from "../../../core/platform/framework/api/crud-service"; import Repository, { comparisonType, inType, @@ -96,13 +100,7 @@ export class DocumentsService { context: DriveExecutionContext & { public_token?: string }, ): Promise => { if (isSharedWithMeFolder(id)) { - const children = await this.search(options, context); - return { - access: "read", - children: children.getEntities(), - nextPage: children.nextPage, - path: [] as Array, - }; + return this.sharedWithMe(options, context); } else { return { nextPage: null, @@ -111,6 +109,25 @@ export class DocumentsService { } }; + sharedWithMe = async ( + options: SearchDocumentsOptions, + context: DriveExecutionContext & { public_token?: string }, + ): Promise => { + const result = []; + let fileList: ListResult; + do { + fileList = await this.search(options, context); + result.push(...fileList.getEntities()); + options.pagination = fileList.nextPage; + } while (fileList.nextPage?.page_token); + return { + access: "read", + children: result, + nextPage: null, + path: [] as Array, + }; + }; + userQuota = async (context: CompanyExecutionContext): Promise => { const children = await this.repository.find({ parent_id: "user_" + context.user.id, @@ -955,9 +972,9 @@ export class DocumentsService { const result = await this.searchRepository.search( {}, { - pagination: { - limitStr: "100", - }, + pagination: options.pagination + ? Pagination.fromPaginable(options.pagination) + : new Pagination(), $in: [ ...(options.onlyDirectlyShared ? [["access_entities", [context.user.id]] as inType] : []), ...(options.company_id ? [["company_id", [options.company_id]] as inType] : []), @@ -999,7 +1016,7 @@ export class DocumentsService { if (!options.onlyDirectlyShared) { const filteredResult = await this.filter(result.getEntities(), async item => { try { - // Check access for each item + //skip all the fiels return await checkAccess(item.id, null, "read", this.repository, context); } catch (error) { this.logger.warn("failed to check item access", error); @@ -1017,7 +1034,6 @@ export class DocumentsService { return new ListResult(result.type, filteredResult, result.nextPage); } - return result; }; diff --git a/tdrive/backend/node/src/services/documents/types.ts b/tdrive/backend/node/src/services/documents/types.ts index 38e14810f..f943c8a9a 100644 --- a/tdrive/backend/node/src/services/documents/types.ts +++ b/tdrive/backend/node/src/services/documents/types.ts @@ -52,6 +52,7 @@ export type SearchDocumentsOptions = { fields?: string[]; onlyDirectlyShared?: boolean; onlyUploadedNotByMe?: boolean; + pagination?: Paginable; }; export type SearchDocumentsBody = { diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 215236a29..e025afeb0 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -6,9 +6,9 @@ import { File } from "../../../../services/files/entities/file"; import { UploadOptions } from "../../../../services/files/types"; import globalResolver from "../../../../services/global-resolver"; import { + CompanyUserRole, PaginationQueryParameters, ResourceWebsocket, - CompanyUserRole, } from "../../../../utils/types"; import { DriveFile } from "../../entities/drive-file"; import { FileVersion } from "../../entities/file-version"; @@ -199,48 +199,10 @@ export class DocumentsController { onlyUploadedNotByMe: true, }; - const data = { + return { ...(await globalResolver.services.documents.documents.browse(id, options, context)), - websockets: request.currentUser?.id - ? globalResolver.platformServices.realtime.sign( - [{ room: `/companies/${context.company.id}/documents/item/${id}` }], - request.currentUser?.id, - ) - : [], + websockets: [], }; - - return data; - }; - - sharedWithMe = async ( - request: FastifyRequest<{ - Params: RequestParams; - Body: SearchDocumentsBody; - Querystring: { public_token?: string }; - }>, - ): Promise> => { - try { - const context = getDriveExecutionContext(request); - - const options: SearchDocumentsOptions = { - ...request.body, - company_id: request.body.company_id || context.company.id, - view: DriveFileDTOBuilder.VIEW_SHARED_WITH_ME, - onlyDirectlyShared: true, - onlyUploadedNotByMe: true, - }; - - if (!Object.keys(options).length) { - this.throw500Search(); - } - - const fileList = await globalResolver.services.documents.documents.search(options, context); - - return this.driveFileDTOBuilder.build(fileList, context, options.fields, options.view); - } catch (error) { - logger.error({ error: `${error}` }, "error while searching for document"); - this.throw500Search(); - } }; /** diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index 3e40a4ce5..70af3cbde 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -107,13 +107,6 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) handler: documentsController.search.bind(documentsController), }); - fastify.route({ - method: "POST", - url: `${baseUrl}/shared-with-me`, - preValidation: [fastify.authenticate], - handler: documentsController.sharedWithMe.bind(documentsController), - }); - fastify.route({ method: "POST", url: `${baseUrl}/browse/:id`, diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index 758cf1e5e..fea26f9b4 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -343,15 +343,15 @@ export default class UserApi { ) { const response = await this.platform.app.inject({ method: "POST", - url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/shared-with-me`, + url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/browse/shared_with_me`, headers: { authorization: `Bearer ${this.jwt}` }, payload }); - return deserialize( - SearchResultMockClass, + return deserialize( + DriveItemDetailsMockClass, response.body); }; diff --git a/tdrive/backend/node/test/e2e/documents/documents-browser.spec.ts b/tdrive/backend/node/test/e2e/documents/documents-browser.spec.ts index 6beaed30a..2742f23ab 100644 --- a/tdrive/backend/node/test/e2e/documents/documents-browser.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/documents-browser.spec.ts @@ -126,10 +126,32 @@ describe("The Documents Browser Window and API", () => { await new Promise(r => setTimeout(r, 3000)); //then file become searchable - expect((await anotherUser.browseDocuments("shared_with_me", {})).children).toHaveLength(1); + expect((await anotherUser.browseDocuments("shared_with_me", {pageSize: 1})).children).toHaveLength(1); }); - }); + it("Share With Me should return all the files that was share by user at one", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, {companyRole: "admin"}); + const anotherUser = await UserApi.getInstance(platform, true, {companyRole: "admin"}); + + let files = await oneUser.uploadAllFilesOneByOne(); + + await anotherUser.uploadAllFilesOneByOne(); + await new Promise(r => setTimeout(r, 5000)); + //give permissions to the file + files[2].access_info.entities.push({ + type: "user", + id: anotherUser.user.id, + level: "read", + grantor: null, + }); + await oneUser.updateDocument(files[2].id, files[2]); + await new Promise(r => setTimeout(r, 3000)); + + //then file become searchable + expect((await anotherUser.browseDocuments("shared_with_me", {pagination: {limitStr: 100}})).children).toHaveLength(1); + }); + }); }); diff --git a/tdrive/backend/node/test/e2e/documents/documents-search.spec.ts b/tdrive/backend/node/test/e2e/documents/documents-search.spec.ts index 47c6689e6..78404b9e4 100644 --- a/tdrive/backend/node/test/e2e/documents/documents-search.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/documents-search.spec.ts @@ -106,7 +106,7 @@ describe("the Drive Search feature", () => { const searchResponse = await userTwo.sharedWithMeDocuments({}); //then:: - expect(searchResponse.entities?.length).toEqual(1); + expect(searchResponse.children?.length).toEqual(1); }) it("did search for an item that doesn't exist", async () => { @@ -146,7 +146,7 @@ describe("the Drive Search feature", () => { it("did search a file shared by another user", async () => { //then file become searchable - expect((await userTwo.sharedWithMeDocuments({})).entities).toHaveLength(1); + expect((await userTwo.sharedWithMeDocuments({})).children).toHaveLength(1); }); it("did search a file by file owner", async () => { diff --git a/tdrive/backend/node/test/http/elasticsearch.http b/tdrive/backend/node/test/http/elasticsearch.http index 280824a0f..e92c16ab3 100644 --- a/tdrive/backend/node/test/http/elasticsearch.http +++ b/tdrive/backend/node/test/http/elasticsearch.http @@ -20,13 +20,61 @@ Content-Type: application/json } } +### Query data +GET https://localhost:9200/drive_files/_search +Authorization: Basic admin admin +Content-Type: application/json + +{ + "query": { + "bool": { + "boost": 1, + "must": [ + { + "bool": { + "should": [ + { + "match": { + "access_entities": { + "query": "7f2e4e80-d4be-11ee-8a11-0544b51cf428", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match": { + "company_id": { + "query": "7efb7eb1-d4be-11ee-8a11-0544b51cf428", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + } + ] + } + } +} ### Check that it is created GET https://localhost:9200/_cat/indices +Authorization: Basic admin admin +Content-Type: application/json ### GET index info -GET https://localhost:9200/drive_files +GET https://localhost:9200/user +Authorization: Basic admin admin +Content-Type: application/json ### GET data to the index GET https://localhost:9200/drive_files/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d