Merge pull request #1049 from rocket-admin/backend_oracle_date_filter

Backend oracle date filtering fixes

backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts

@@ -4,11 +4,16 @@ import { faker } from '@faker-js/faker';
 import { INestApplication, ValidationPipe } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
 import test from 'ava';
+import { ValidationError } from 'class-validator';
 import cookieParser from 'cookie-parser';
+import fs from 'fs';
+import path, { join } from 'path';
 import request from 'supertest';
+import { fileURLToPath } from 'url';
 import { ApplicationModule } from '../../../src/app.module.js';
 import { LogOperationTypeEnum, QueryOrderingEnum } from '../../../src/enums/index.js';
 import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js';
+import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js';
 import { Messages } from '../../../src/exceptions/text/messages.js';
 import { Cacher } from '../../../src/helpers/cache/cacher.js';
 import { Constants } from '../../../src/helpers/constants/constants.js';
@@ -20,14 +25,6 @@ import { dropTestTables } from '../../utils/drop-test-tables.js';
 import { getTestData } from '../../utils/get-test-data.js';
 import { registerUserAndReturnUserInfo } from '../../utils/register-user-and-return-user-info.js';
 import { TestUtils } from '../../utils/test.utils.js';
-import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js';
-import { ValidationError } from 'class-validator';
-import knex, { Knex } from 'knex';
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import { join } from 'path';
-import oracledb from 'oracledb';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
@@ -1349,6 +1346,81 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC
   },
 );
 
+test.serial(`${currentTest} with pagination, with sorting and with filtering by date fields
+should return all found rows with search, pagination: page=1, perPage=2 and DESC sorting and filtering`, async (t) => {
+  try {
+    const connectionToTestDB = getTestData(mockFactory).connectionToOracleDB;
+    const firstUserToken = (await registerUserAndReturnUserInfo(app)).token;
+    const { testTableName, testTableColumnName } = await createTestOracleTable(connectionToTestDB);
+
+    testTables.push(testTableName);
+
+    const createConnectionResponse = await request(app.getHttpServer())
+      .post('/connection')
+      .send(connectionToTestDB)
+      .set('Cookie', firstUserToken)
+      .set('Content-Type', 'application/json')
+      .set('Accept', 'application/json');
+    const createConnectionRO = JSON.parse(createConnectionResponse.text);
+    t.is(createConnectionResponse.status, 201);
+
+    const createTableSettingsDTO = mockFactory.generateTableSettings(
+      createConnectionRO.id,
+      testTableName,
+      [testTableColumnName],
+      undefined,
+      undefined,
+      3,
+      QueryOrderingEnum.DESC,
+      'id',
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+    );
+
+    const firstFieldName = 'created_at';
+    const secondFieldName = 'updated_at';
+    const firstFieldValue = "2011-11-03";
+
+    const filters = {
+      [firstFieldName]: { lt: firstFieldValue },
+    };
+
+    const getTableRowsResponse = await request(app.getHttpServer())
+      .post(`/table/rows/find/${createConnectionRO.id}?tableName=${testTableName}&page=1&perPage=2`)
+      .send({ filters })
+      .set('Cookie', firstUserToken)
+      .set('Content-Type', 'application/json')
+      .set('Accept', 'application/json');
+
+    const getTableRowsRO = JSON.parse(getTableRowsResponse.text);
+    t.is(getTableRowsResponse.status, 201);
+    t.is(typeof getTableRowsRO, 'object');
+    t.is(getTableRowsRO.hasOwnProperty('rows'), true);
+    t.is(getTableRowsRO.hasOwnProperty('primaryColumns'), true);
+    t.is(getTableRowsRO.hasOwnProperty('pagination'), true);
+    t.is(getTableRowsRO.rows.length, 2);
+    t.is(Object.keys(getTableRowsRO.rows[1]).length, 5);
+
+    t.is(getTableRowsRO.rows[0][testTableColumnName], testSearchedUserName);
+    t.is(getTableRowsRO.rows[0].id, 1);
+    t.is(getTableRowsRO.rows[1][testTableColumnName], testSearchedUserName);
+    t.is(getTableRowsRO.rows[1].id, 22);
+
+    t.is(getTableRowsRO.pagination.currentPage, 1);
+    t.is(getTableRowsRO.pagination.perPage, 2);
+    t.is(typeof getTableRowsRO.primaryColumns, 'object');
+    t.is(getTableRowsRO.primaryColumns[0].hasOwnProperty('column_name'), true);
+
+    // t.is(getTableRowsRO.primaryColumns[0].hasOwnProperty('data_type'), true);
+  } catch (e) {
+    console.error(e);
+    throw e;
+  }
+});
+
 test.serial(
   `${currentTest} with search, with pagination, with sorting and with filtering
 should return all found rows with search, pagination: page=1, perPage=10 and DESC sorting and filtering'`,

backend/test/utils/create-test-table.ts

@@ -325,7 +325,7 @@ export async function createTestOracleTable(
           [pColumnName]: ++counter,
           [testTableColumnName]: testSearchedUserName,
           [testTableSecondColumnName]: faker.internet.email(),
-          created_at: new Date(),
+          created_at: new Date("2010-11-03"),
           updated_at: new Date(),
         });
       } else {

shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts

@@ -72,6 +72,44 @@ export class BasicDataAccessObject {
     }
   }
 
+  protected isDateTimeType(columnTypeName: string): boolean {
+    const dateTimeDataTypes = [
+      // PostgreSQL
+      'DATE',
+      'TIME',
+      'TIMETZ',
+      'TIMESTAMP',
+      'TIMESTAMPTZ',
+
+      // MySQL
+      'DATE',
+      'DATETIME',
+      'TIMESTAMP',
+      'TIME',
+      'YEAR',
+
+      // MS SQL Server
+      'DATE',
+      'DATETIME',
+      'DATETIME2',
+      'DATETIMEOFFSET',
+      'SMALLDATETIME',
+      'TIME',
+
+      // OracleDB
+      'DATE',
+      'TIMESTAMP',
+      'TIMESTAMP WITH TIME ZONE',
+      'TIMESTAMP WITH LOCAL TIME ZONE',
+
+      // IBM Db2
+      'DATE',
+      'TIME',
+      'TIMESTAMP',
+    ];
+    return dateTimeDataTypes.includes(columnTypeName.toUpperCase());
+  }
+
   private isValidName(name: string): boolean {
     return typeof name === 'string' && name.length > 0 && /^[a-zA-Z0-9_]+$/.test(name);
   }

shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts

@@ -1,7 +1,20 @@
 /* eslint-disable security/detect-object-injection */
+import * as csv from 'csv';
 import { Knex } from 'knex';
+import { Readable, Stream } from 'node:stream';
+import { LRUStorage } from '../../caching/lru-storage.js';
 import { checkFieldAutoincrement } from '../../helpers/check-field-autoincrement.js';
 import { DAO_CONSTANTS } from '../../helpers/data-access-objects-constants.js';
+import { ERROR_MESSAGES } from '../../helpers/errors/error-messages.js';
+import {
+  isOracleDateOrTimeType,
+  isOracleDateStringByRegexp,
+  isOracleDateType,
+  isOracleTimeType,
+} from '../../helpers/is-database-date.js';
+import { objectKeysToLowercase } from '../../helpers/object-kyes-to-lowercase.js';
+import { renameObjectKeyName } from '../../helpers/rename-object-keyname.js';
+import { tableSettingsFieldValidator } from '../../helpers/validation/table-settings-validator.js';
 import { AutocompleteFieldsDS } from '../shared/data-structures/autocomplete-fields.ds.js';
 import { ConnectionParams } from '../shared/data-structures/connections-params.ds.js';
 import { FilteringFieldsDS } from '../shared/data-structures/filtering-fields.ds.js';
@@ -11,20 +24,12 @@ import { PrimaryKeyDS } from '../shared/data-structures/primary-key.ds.js';
 import { ReferencedTableNamesAndColumnsDS } from '../shared/data-structures/referenced-table-names-columns.ds.js';
 import { TableSettingsDS } from '../shared/data-structures/table-settings.ds.js';
 import { TableStructureDS } from '../shared/data-structures/table-structure.ds.js';
+import { TableDS } from '../shared/data-structures/table.ds.js';
 import { TestConnectionResultDS } from '../shared/data-structures/test-result-connection.ds.js';
 import { ValidateTableSettingsDS } from '../shared/data-structures/validate-table-settings.ds.js';
 import { FilterCriteriaEnum } from '../shared/enums/filter-criteria.enum.js';
 import { IDataAccessObject } from '../shared/interfaces/data-access-object.interface.js';
 import { BasicDataAccessObject } from './basic-data-access-object.js';
-import { LRUStorage } from '../../caching/lru-storage.js';
-import { objectKeysToLowercase } from '../../helpers/object-kyes-to-lowercase.js';
-import { renameObjectKeyName } from '../../helpers/rename-object-keyname.js';
-import { tableSettingsFieldValidator } from '../../helpers/validation/table-settings-validator.js';
-import { TableDS } from '../shared/data-structures/table.ds.js';
-import { ERROR_MESSAGES } from '../../helpers/errors/error-messages.js';
-import { Stream, Readable } from 'node:stream';
-import * as csv from 'csv';
-import { isOracleDateOrTimeType, isOracleDateStringByRegexp } from '../../helpers/is-database-date.js';
 
 type RefererencedConstraint = {
   TABLE_NAME: string;
@@ -232,6 +237,13 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
 
     const tableStructure = await this.getTableStructure(tableName);
     const availableFields = this.findAvailableFields(settings, tableStructure);
+    const timestampColumnNames = tableStructure
+      .filter(({ data_type }) => isOracleTimeType(data_type))
+      .map(({ column_name }) => column_name);
+
+    const datesColumnsNames = tableStructure
+      .filter(({ data_type }) => isOracleDateType(data_type))
+      .map(({ column_name }) => column_name);
 
     const searchedFields =
       settings?.search_fields?.length > 0 ? settings.search_fields : searchedFieldValue ? availableFields : [];
@@ -248,6 +260,8 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
       searchedFieldValue,
       filteringFields,
       settings,
+      timestampColumnNames,
+      datesColumnsNames,
     );
   }
 
@@ -260,8 +274,10 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
     availableFields: Array<string>,
     searchedFields: Array<string>,
     searchedFieldValue: any,
-    filteringFields: any,
+    filteringFields: FilteringFieldsDS[],
     settings: TableSettingsDS,
+    timestampColumnNames: Array<string>,
+    datesColumnsNames: Array<string>,
   ) {
     const offset = (page - 1) * perPage;
 
@@ -277,9 +293,21 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
       }
     };
 
-    const applyFilteringFields = (builder: Knex.QueryBuilder) => {
+    const applyFilteringFields = (
+      builder: Knex.QueryBuilder,
+      timestampColumnNames: Array<string>,
+      datesColumnsNames: Array<string>,
+    ) => {
       if (filteringFields && filteringFields.length > 0) {
-        for (const { field, criteria, value } of filteringFields) {
+        // eslint-disable-next-line prefer-const
+        for (let { field, criteria, value } of filteringFields) {
+          if (datesColumnsNames.includes(field)) {
+            const valueToDate = new Date(String(value));
+            value = this.formatDate(valueToDate);
+          }
+          if (timestampColumnNames.includes(field)) {
+            value = this.formatTimestamp(String(value));
+          }
           const operators = {
             [FilterCriteriaEnum.eq]: '=',
             [FilterCriteriaEnum.startswith]: 'like',
@@ -314,7 +342,7 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
       .withSchema(tableSchema)
       .select(availableFields)
       .modify(applySearchFields)
-      .modify(applyFilteringFields)
+      .modify((builder) => applyFilteringFields(builder, timestampColumnNames, datesColumnsNames))
       .modify(applyOrdering)
       .limit(perPage)
       .offset(offset);
@@ -770,4 +798,24 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa
     const resultString = `${day}-${monthNames[monthIndex]}-${year}`;
     return resultString;
   }
+
+  private formatTimestamp(timestamp: string | number | Date): string {
+    const date = new Date(timestamp);
+
+    const day = `0${date.getDate()}`.slice(-2);
+    const monthNames = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC'];
+    const month = monthNames[date.getMonth()];
+    const year = date.getFullYear().toString().slice(-2);
+
+    let hours = date.getHours();
+    const period = hours >= 12 ? 'PM' : 'AM';
+    hours = hours % 12;
+    hours = hours ? hours : 12;
+    const hoursStr = `0${hours}`.slice(-2);
+
+    const minutes = `0${date.getMinutes()}`.slice(-2);
+    const seconds = `0${date.getSeconds()}`.slice(-2);
+
+    return `${day}-${month}-${year} ${hoursStr}:${minutes}:${seconds} ${period}`;
+  }
 }

shared-code/src/helpers/is-database-date.ts

@@ -18,10 +18,31 @@ export function isOracleDateOrTimeType(type: string): boolean {
   if (type.toLowerCase().includes('timestamp')) {
     return true;
   }
-  const dateTypes = ['date', 'timestamp', 'timestamp with time zone', 'timestamp with local time zone'];
+  const dateTypes = [
+    'date',
+    'timestamp',
+    'timestamp with time zone',
+    'timestamp with local time zone',
+    'timestamp(6) with local time zone',
+    'timestamp(0) with local time zone',
+  ];
   return dateTypes.includes(type.toLowerCase());
 }
 
+export function isOracleTimeType(type: string): boolean {
+  return [
+    'timestamp',
+    'timestamp with time zone',
+    'timestamp with local time zone',
+    'timestamp(6) with local time zone',
+    'timestamp(0) with local time zone',
+  ].includes(type.toLowerCase());
+}
+
+export function isOracleDateType(type: string): boolean {
+  return ['date'].includes(type.toLowerCase());
+}
+
 export function isOracleDateStringByRegexp(value: string): boolean {
   const dateRegexp = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/;
   return dateRegexp.test(value);