Skip to content

Commit

Permalink
feat(condo): DOMA-10613 made parsing for similar date formats and sup…
Browse files Browse the repository at this point in the history 
…port for YYYYMM MMYYYY formats
  • Loading branch information
@YEgorLu
YEgorLu committed Nov 21, 2024
1 parent 174e9c3 commit cf35821
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 26 deletions.
23 changes: 22 additions & 1 deletion apps/condo/domains/common/constants/import.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
const dayjs = require('dayjs')

const VALID_YEAR_OFFSET_FROM_TODAY = 30

const PROCESSING = 'processing'
const COMPLETED = 'completed'
const ERROR = 'error'
Expand Down Expand Up @@ -35,17 +39,34 @@ const DATE_TIME_FORMATS = DATE_FORMATS
TIME_VARIANTS.map(timeVariant => [dateFormat, timeVariant].join(' '))
)

/** @param {dayjs.Dayjs} date */
function yearAroundToday (date) {
return Math.abs(date.year() - dayjs().year()) < VALID_YEAR_OFFSET_FROM_TODAY
}

/** @type {(string | {format:string, validate:(validDate: dayjs.Dayjs)=>boolean})[]} */
const DEFAULT_DATE_PARSING_FORMATS = [
...DATE_TIME_FORMATS,
...DATE_FORMATS,

{ format: 'YYYYMM', validate: yearAroundToday },
{ format: 'MMYYYY', validate: yearAroundToday },

'YYYY-MM-DDTHH:mm:ss.SSS[Z]', // The result of dayjs().toISOString()
'YYYY-MM-DDTHH:mm:ss.SSSZ',
'YYYY-MM-DDTHH:mm:ss.SSS',
'YYYY-MM-DDTHH:mm:ssZZ',
'YYYY-MM-DDTHH:mm:ssZ',
'YYYY-MM-DDTHH:mm:ss',
].sort((a, b) => b.length - a.length) // Order matters! see "Differences to moment" https://day.js.org/docs/en/parse/string-format
].sort((a, b) => {
if (typeof a === 'string' && typeof b === 'string') {
return b.length - a.length
}
if (typeof a === 'string' || typeof b === 'string') {
return typeof a === 'string' ? -1 : 1
}
return b.format.length - a.format.length
}) // Order matters! see "Differences to moment" https://day.js.org/docs/en/parse/string-format
const ISO_DATE_FORMAT = 'YYYY-MM-DD'
const EUROPEAN_DATE_FORMAT = 'DD.MM.YYYY'

Expand Down
71 changes: 56 additions & 15 deletions apps/condo/domains/common/utils/import/date.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,21 @@ dayjs.extend(utc)

const SYMBOLS_TO_CUT_FROM_DATES_REGEXP = /[^+\-TZ_:.,( )/\d]/gi // -_:.,()/ recognizable by dayjs https://day.js.org/docs/en/parse/string-format
const DATE_WITH_OFFSET_REGEXP = /[+-](\d{2}):?(\d{2})$/ // +5000 | +50:00
const UTC_FORMAT_ENDING = 'Z]'
const OFFSET_FORMAT_ENDING = 'Z' // ...Z or ...ZZ

function isUtcFormat (format) {
return format.endsWith('Z]')
if (typeof format === 'string') {
return format.endsWith(UTC_FORMAT_ENDING)
}
return format.format.endsWith(UTC_FORMAT_ENDING)
}

function isOffsetFormat (format) {
return format.endsWith('Z') // ...Z or ...ZZ
if (typeof format === 'string') {
return format.endsWith(OFFSET_FORMAT_ENDING)
}
return format.format.endsWith(OFFSET_FORMAT_ENDING)
}

/** true if date has ending like +0000 +00:00 -00:00...
Expand Down Expand Up @@ -70,31 +78,27 @@ function clearDateStr (dateStr) {
/**
* Validates date strings
* @param {string} dateStr
* @param {Array<string>?} formats - date parsing formats. Must be not empty array
* @param {(string | {format: string, validate: (date: dayjs.Dayjs) => boolean })[]?} formats - date parsing formats. Must be not empty array
* @return {boolean}
*/
function isDateStrValid (dateStr, formats = DEFAULT_DATE_PARSING_FORMATS) {
if (!dateStr || !isString(dateStr)) {
return false
}

const parseDayjs = (dateString, formats) => tryParseDateWithRestrictions(dayjs, dateString, formats, true)

// dayjs does recognize these formats, so no additional manipulations required
if (dayjs(dateStr, formats, true).isValid()) {
if (parseDayjs(dateStr, formats).isValid()) {
return true
}

if (isOffsetDate(dateStr)) {
let offsetFormats = formats.filter(format => isOffsetFormat(format))
if (!offsetFormats.length) {
return dayjs(dateStr, formats, true).isValid()
}
if (dayjs(dateStr, formats, true).isValid()) {
return true
}

// we need to pass dateStr to dayjs in strict mode to check valid values in MM, DD, HH, mm...
// but dayjs can not parse this format strictly https://github.com/iamkun/dayjs/issues/929
// so lets manually drop offsets from date string and format and check what's left
// so lets manually drop offsets from date string and format and check what's leftxx§

const maybeSemicolon = dateStr[dateStr.length - 3]
let offsetStartIndex = dateStr.length - 5
Expand All @@ -107,16 +111,53 @@ function isDateStrValid (dateStr, formats = DEFAULT_DATE_PARSING_FORMATS) {
return format.substring(0, format.length - offsetFromEnd)
})

return dayjs(dateStr, offsetFormats, true).isValid()
return parseDayjs(dateStr, offsetFormats).isValid()
}

return false
}

/**
* @param {(dateString: string, formats: string | string[], strict: boolean) => dayjs.Dayjs} dayjsFunc
* @param dateString
* @param {({format: string, validate: (date: dayjs.Dayjs) => boolean} | string)[]} formats
* @param strict
* @returns dayjs.Dayjs
* */
function tryParseDateWithRestrictions (dayjsFunc, dateString, formats, strict) {
const { formatsWithRestrictions, simpleFormats } = formats.reduce((acc, format) => {
if (typeof format !== 'string') {
acc.formatsWithRestrictions.push(format)
} else {
acc.simpleFormats.push(format)
}
return acc
}, { formatsWithRestrictions: [], simpleFormats: [] })

if (simpleFormats.length) {
const simpleFormatDate = dayjsFunc(dateString, simpleFormats, strict)
if (simpleFormatDate.isValid()) {
return simpleFormatDate
}
}

for (const format of formatsWithRestrictions) {
const date = dayjsFunc(dateString, format.format, strict)
if (!date.isValid()) {
continue
}
if (format.validate(date)) {
return date
}
}

return dayjs('Invalid date')
}

/**
* Parses strings in UTC format always. Extra formats can be passed.
* @param {string} dateStr
* @param {Array<string>?} overrideFormats - valid input formats. Must be not empty array
* @param {(string | {format:string, validate:(validDate:dayjs.Dayjs)=>boolean})[]?} overrideFormats - valid input formats. Must be not empty array
* @returns {string|undefined} dateString in UTC or undefined for invalid date
*/
function tryToISO (dateStr, overrideFormats = DEFAULT_DATE_PARSING_FORMATS) {
Expand All @@ -128,9 +169,9 @@ function tryToISO (dateStr, overrideFormats = DEFAULT_DATE_PARSING_FORMATS) {
let date
if (dateStr.endsWith('Z')) {
const utcStringFormats = overrideFormats.filter(format => isUtcFormat(format))
date = tryParseUtcDate(dateStr, utcStringFormats, false)
date = tryParseDateWithRestrictions(tryParseUtcDate, dateStr, utcStringFormats, false)
} else {
date = dayjs(dateStr, overrideFormats, false)
date = tryParseDateWithRestrictions(dayjs, dateStr, overrideFormats, false)
}
// undefined needed to not update invalid dates to null
return date.isValid() ? date.toISOString() : undefined
Expand Down
16 changes: 16 additions & 0 deletions apps/condo/domains/common/utils/import/date.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ const validDateStrings = [
'01-01-2024',
'2024-01-01 00:00:00',
'2024-01-01 00:00',
'092009', // 2009-09-01...
'200012', // 2000-12-01...
]

const invalidDateStrings = [
Expand All @@ -21,6 +23,8 @@ const invalidDateStrings = [
'invalid-date', // Not a date
'2024/13/01', // Invalid month
'2024-01-01T25:00:00Z', // Invalid hour
'091000', // 1000-09-01... technically
'100009', // 1000-09-01... technically
]

describe('importDate.utils', () => {
Expand Down Expand Up @@ -96,6 +100,18 @@ describe('importDate.utils', () => {
expect(tryToISO(date, utcFormats)).toBe(expectedDate)
})
})

describe('Works with simmilar formats if validator provided', () => {
const yearIsNearToday = (date) => Math.abs(date.year() - new Date().getFullYear()) < 30
const cases = [
{ similarFormats: [{ format: 'YYYYMM', validate: yearIsNearToday }, { format: 'MMYYYY', validate: yearIsNearToday }], dateString: '092009', expectedDate: '2009-08-31T18:00:00.000Z' },
{ similarFormats: [{ format: 'YYYYMM', validate: yearIsNearToday }, { format: 'MMYYYY', validate: yearIsNearToday }], dateString: '200909', expectedDate: '2009-08-31T18:00:00.000Z' },
]

it.each(cases)('$similarFormats.0.format - $similarFormats.1.format $dateString', ({ similarFormats, dateString, expectedDate }) => {
expect(tryToISO(dateString, similarFormats)).toBe(expectedDate)
})
})
})

})
34 changes: 25 additions & 9 deletions apps/condo/domains/meter/tasks/importMeters.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const path = require('path')

const index = require('@app/condo/index')
const { faker } = require('@faker-js/faker')
const dayjs = require('dayjs')
const { get } = require('lodash')
const XLSX = require('xlsx')

Expand All @@ -19,6 +20,7 @@ const {
DOMA_EXCEL,
CANCELLED,
ERROR,
DEFAULT_DATE_PARSING_FORMATS,
} = require('@condo/domains/common/constants/import')
const { EXCEL_FILE_META } = require('@condo/domains/common/utils/createExportFile')
const { readXlsx, getTmpFile, downloadFile } = require('@condo/domains/common/utils/testSchema/file')
Expand All @@ -45,19 +47,22 @@ const readMockFile = (fileName) => {
return fs.readFileSync(path.join(__dirname, MOCK_FOLDER, fileName))
}

const generateCsvFile = (validLinesSize, invalidLinesSize, fatalErrorLinesSize, property) => {
const generateCsvFile = (validLinesSize, invalidLinesSize, fatalErrorLinesSize, property, dateFormats = DEFAULT_DATE_PARSING_FORMATS) => {
// content header
let content = `#DATE_BEGIN01.12.2023
#DATE_END: 31.12.2023`
const defaultDate = dayjs('31.12.2023', 'DD.MM.YYYY')

// generate valid lines lines
for (let i = 0; i < validLinesSize; i++) {
const number = faker.datatype.number({ min: 1000, max: 9999 })
const lastName = faker.name.lastName()
const unitName = `${i + 1}`
const address = property.address + ', кв ' + unitName
const randomFormat = faker.helpers.arrayElement(dateFormats)
const dateString = defaultDate.format(typeof randomFormat === 'string' ? randomFormat : randomFormat.format)
content += `
00-00000${number};${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;
00-00000${number};${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;${dateString};;;;;;;${dateString};;;;;;;${dateString};;;;;;;${dateString};
`
}

Expand All @@ -67,8 +72,10 @@ const generateCsvFile = (validLinesSize, invalidLinesSize, fatalErrorLinesSize,
const lastName = faker.name.lastName()
const unitName = `${validLinesSize + i + 1}`
const address = property.address + ', кв ' + unitName
const randomFormat = faker.helpers.arrayElement(dateFormats)
const dateString = defaultDate.format(typeof randomFormat === 'string' ? randomFormat : randomFormat.format)
content += `
;${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;
;${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;${dateString};;;;;;;${dateString};;;;;;;${dateString};;;;;;;${dateString};
`
}

Expand All @@ -77,43 +84,52 @@ const generateCsvFile = (validLinesSize, invalidLinesSize, fatalErrorLinesSize,
const number = faker.datatype.number({ min: 1000, max: 9999 })
const lastName = faker.name.lastName()
const address = faker.address.streetAddress(true)
const randomFormat = faker.helpers.arrayElement(dateFormats)
const dateString = defaultDate.format(typeof randomFormat === 'string' ? randomFormat : randomFormat.format)
content += `
;${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;;;;;;;31.12.2023;
;${lastName} Л.М.;40ОН89${number}-02;${faker.datatype.uuid()};${address};9;${number}${number};ХВС;[];750,00;${dateString};;;;;;;${dateString};;;;;;;${dateString};;;;;;;${dateString};
`
}

return createUpload(content, `${faker.datatype.uuid()}.csv`, 'text/csv')
}

const generateExcelFile = async (validLinesSize, invalidLinesSize, fatalLinesSize, property) => {
const generateExcelFile = async (validLinesSize, invalidLinesSize, fatalLinesSize, property, dateFormats = DEFAULT_DATE_PARSING_FORMATS) => {
const data = [[
'Адрес', 'Помещение', 'Тип помещения', 'Лицевой счет',
'Тип счетчика', 'Номер счетчика', 'Количество тарифов',
'Показание 1', 'Показание 2', 'Показание 3', 'Показание 4',
'Дата передачи показаний', 'Дата поверки', 'Дата следующей поверки',
'Дата установки', 'Дата ввода в эксплуатацию', 'Дата опломбирования', 'Дата контрольных показаний', 'Место установки счетчика', 'Автоматический']]

const defaultDates = ['2021-01-21', '2021-01-21', '2021-01-21',
'2021-01-22', '2021-01-23', '2021-01-24'].map(dateString => dayjs(dateString, 'YYYY-MM-DD'))

for (let i = 0; i < validLinesSize; i++) {
const unitName = `${i + 1}`
const randomFormat = faker.helpers.arrayElement(dateFormats)
const dateStrings = defaultDates.map(date => date.format(typeof randomFormat === 'string' ? randomFormat : randomFormat.format))
const line = [
property.address, unitName, 'Квартира', `${faker.datatype.number({ min: 1000, max: 9999 })}`,
'ГВС', `${faker.datatype.number({ min: 1000, max: 9999 })}`, '1',
`${faker.datatype.number({ min: 1000, max: 9999 })}`, '', '', '',
'2021-01-21', '2021-01-21', '2021-01-21',
'2021-01-22', '2021-01-23', '2021-01-24', '2021-01-25', 'Кухня', '',
dateStrings[0], dateStrings[1], dateStrings[2],
dateStrings[3], dateStrings[4], dateStrings[5], dateStrings[6], 'Кухня', '',
]

data.push(line)
}

for (let i = 0; i < invalidLinesSize; i++) {
const unitName = `${i + 1}`
const randomFormat = faker.helpers.arrayElement(dateFormats)
const dateStrings = defaultDates.map(date => date.format(typeof randomFormat === 'string' ? randomFormat : randomFormat.format))
const line = [
property.address, unitName, 'Квартира', `${faker.datatype.number({ min: 1000, max: 9999 })}`,
'WRONG_METER_TYPE', `${faker.datatype.number({ min: 1000, max: 9999 })}`, '1',
`${faker.datatype.number({ min: 1000, max: 9999 })}`, '', '', '',
'2021-01-21', '2021-01-21', '2021-01-21',
'2021-01-22', '2021-01-23', '2021-01-24', '2021-01-25', 'Кухня',
dateStrings[0], dateStrings[1], dateStrings[2],
dateStrings[3], dateStrings[4], dateStrings[5], dateStrings[6], 'Кухня',
]

data.push(line)
Expand Down
7 changes: 6 additions & 1 deletion apps/condo/domains/meter/utils/taskSchema/AbstractMetersImporter.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,12 @@ class ImporterWrapper extends AbstractMetersImporter {


function generateValidDatesByFormats (formats) {
return formats.map(format => dayjs(faker.date.past()).format(format))
return formats.map(format => {
if (typeof format !== 'string') {
format = format.format
}
return dayjs(faker.date.past()).format(format)
})
}

const defaultReading = {
Expand Down

0 comments on commit cf35821

Please sign in to comment.