diff --git a/__tests__/actions/forms/fields/birdsNewSpeciesModeratorReview.js b/__tests__/actions/forms/fields/birdsNewSpeciesModeratorReview.js index e1986bfc6..3144ae0af 100644 --- a/__tests__/actions/forms/fields/birdsNewSpeciesModeratorReview.js +++ b/__tests__/actions/forms/fields/birdsNewSpeciesModeratorReview.js @@ -43,7 +43,7 @@ describe('birdsNewSpeciesModeratorReview', () => { await bgatlasSpeciesFactory(setup.api, cell, species) const { id } = await formFactory(setup.api, { - species: species, + species, user: user.email, ...getCenter(cell.coordinates()) }) @@ -89,13 +89,13 @@ describe('birdsNewSpeciesModeratorReview', () => { const species = await speciesFactory(setup.api, 'birds') const { id: record1 } = await formFactory(setup.api, { - species: species, + species, user: user.email, ...getCenter(cell.coordinates()) }) const { id: record2 } = await formFactory(setup.api, { - species: species, + species, user: user.email, ...getCenter(cell.coordinates()) }) diff --git a/__tests__/actions/import.js b/__tests__/actions/import.js index 0b4b8da09..c8ce93486 100644 --- a/__tests__/actions/import.js +++ b/__tests__/actions/import.js @@ -2,6 +2,8 @@ /* globals setup */ const forms = require('../../__utils__/forms.js') +const userFactory = require('../../__utils__/factories/userFactory') +const _ = require('lodash') const formsWithGenerators = forms.map((form) => { return { @@ -10,6 +12,7 @@ const formsWithGenerators = forms.map((form) => { } }) +// Temporary skip all tests describe('Import', () => { afterEach(() => { return Promise.all(forms.map(form => { @@ -20,18 +23,29 @@ describe('Import', () => { })) }) - setup.describeAsAuth((runAction) => { - test.each(formsWithGenerators.map(form => [form.name, form]))('imports %s records', async (name, form) => { - const records = [] - for (let i = 0; i < 5; i++) { - const recordData = await form.generator(setup.api, { notes: 'from import' }, { create: false }) - records.push(recordData) - } - - return runAction(`${form.name}:import`, { items: records }).then(function (response) { - return setup.api.models[form.name].findAll({ where: { notes: 'from import' } }).then(function (items) { - expect(items.length).toBe(5) - }) + const roles = ['user', 'admin'] + + roles.forEach((role) => { + setup[`describeAs${_.capitalize(role.toLowerCase())}`]((specs) => { + test.each(formsWithGenerators.map(form => [form.name, form]))('imports %s records', async (name, form) => { + const user = await userFactory(setup.api, { role }) + + const records = [] + for (let i = 0; i < 5; i++) { + const recordData = await form.generator(setup.api, { notes: 'from import' }, { create: false, apiInsertFormat: true }) + records.push(recordData) + } + + await setup.api.tasks.tasks['form:import'].run( + { + params: { items: records, user: user.id, language: 'en' }, + user, + formName: form.name + } + ) + + const expectedItems = await setup.api.models[form.name].findAll({ where: { notes: 'from import' } }) + expect(expectedItems.length).toBe(5) }) }) }) diff --git a/__tests__/actions/session.js b/__tests__/actions/session.js index a04b66958..7900230a7 100644 --- a/__tests__/actions/session.js +++ b/__tests__/actions/session.js @@ -91,7 +91,7 @@ describe('Action session', () => { conn = await setup.api.specHelper.Connection.createAsync() conn.params = { email: user.email, - password: password + password } csrfToken = await setup.runAction('session:create', conn) diff --git a/__tests__/tasks/formExport.js b/__tests__/tasks/formExport.js index 2c643f2da..95432137e 100644 --- a/__tests__/tasks/formExport.js +++ b/__tests__/tasks/formExport.js @@ -23,7 +23,7 @@ test.each([ }, formName, outputType: 'csv', - user: user + user }) expect(api.tasks.enqueue).toHaveBeenCalledWith('mail:send', { diff --git a/__utils__/factories/formBirdsFactory.js b/__utils__/factories/formBirdsFactory.js index cd1ab014c..50797c30d 100644 --- a/__utils__/factories/formBirdsFactory.js +++ b/__utils__/factories/formBirdsFactory.js @@ -9,14 +9,24 @@ async function formBirdsFactory (api, { countMax = count, ...otherProps }, { - create = true + create = true, + apiInsertFormat = false } = {}) { species = await species + const record = { ...await formCommonFactory(api, otherProps), species: species.labelLa || species, - ...localFieldFactory('countUnit'), - ...localFieldFactory('typeUnit'), + ...await localFieldFactory(api, 'birds_count_units', 'countUnit', + { + apiInsertFormat + } + ), + ...await localFieldFactory(api, 'birds_count_type', 'typeUnit', + { + apiInsertFormat + } + ), count, countMin, countMax, diff --git a/__utils__/factories/formBirdsMigrationsFactory.js b/__utils__/factories/formBirdsMigrationsFactory.js index 718f1cd27..b68400eca 100644 --- a/__utils__/factories/formBirdsMigrationsFactory.js +++ b/__utils__/factories/formBirdsMigrationsFactory.js @@ -9,7 +9,8 @@ async function formBirdsMigrationsFactory (api, { migrationPoint = poisFactory(api, { type: 'birds_migration_point' }), ...otherProps } = {}, { - create = true + create = true, + apiInsertFormat = false } = {}) { species = await species migrationPoint = await migrationPoint @@ -17,7 +18,7 @@ async function formBirdsMigrationsFactory (api, { ...await formCommonFactory(api, otherProps), species: species.labelLa || species, count, - ...localFieldFactory('migrationPoint', { en: migrationPoint.labelEn }), + ...await localFieldFactory(api, 'birds_migration_point', 'migrationPoint', { en: migrationPoint.labelEn, apiInsertFormat }), ...otherProps } diff --git a/__utils__/factories/formCBMFactory.js b/__utils__/factories/formCBMFactory.js index 1fe06c94b..e0bfe5506 100644 --- a/__utils__/factories/formCBMFactory.js +++ b/__utils__/factories/formCBMFactory.js @@ -9,15 +9,17 @@ async function formCBMFactory (api, { zone = zoneFactory(api), ...otherProps } = {}, { - create = true + create = true, + apiInsertFormat = false } = {}) { species = await species zone = await zone const record = { ...await formCommonFactory(api, otherProps), - ...localFieldFactory('distance'), + ...await localFieldFactory(api, 'cbm_distance', 'distance', { apiInsertFormat }), species: species.labelLa || species, count, + ...(apiInsertFormat ? { zone: zone.id } : { zoneId: zone.id }), zoneId: zone.id, ...otherProps } diff --git a/__utils__/factories/formHerptilesFactory.js b/__utils__/factories/formHerptilesFactory.js index 76f93cd38..6992f8b49 100644 --- a/__utils__/factories/formHerptilesFactory.js +++ b/__utils__/factories/formHerptilesFactory.js @@ -2,7 +2,7 @@ const formCommonFactory = require('./formCommonFactory') const speciesFactory = require('./speciesFactory') async function formHerptilesFactory (api, { - species = speciesFactory(api, 'herptiles'), + species = speciesFactory(api, 'herptiles_name'), count = 1, ...otherProps } = {}, { diff --git a/__utils__/factories/formInvertebratesFactory.js b/__utils__/factories/formInvertebratesFactory.js index e24ae0a96..158146186 100644 --- a/__utils__/factories/formInvertebratesFactory.js +++ b/__utils__/factories/formInvertebratesFactory.js @@ -2,7 +2,7 @@ const formCommonFactory = require('./formCommonFactory') const speciesFactory = require('./speciesFactory') async function formInvertebratesFactory (api, { - species = speciesFactory(api, 'invertebrates'), + species = speciesFactory(api, 'invertebrates_name'), count = 1, ...otherProps } = {}, { diff --git a/__utils__/factories/formThreatsFactory.js b/__utils__/factories/formThreatsFactory.js index 9dcfda4a4..ac114113f 100644 --- a/__utils__/factories/formThreatsFactory.js +++ b/__utils__/factories/formThreatsFactory.js @@ -8,15 +8,17 @@ async function formThreatsFactory (api, { primaryType = 'threat', ...otherProps } = {}, { - create = true + create = true, + apiInsertFormat = false } = {}) { species = await species const record = { ...await formCommonFactory(api, otherProps), species: species.labelLa || species, + class: 'mammals', count, primaryType, - ...localFieldFactory('category'), + ...await localFieldFactory(api, 'threats_category', 'category', { apiInsertFormat }), ...otherProps } diff --git a/__utils__/factories/localFieldFactory.js b/__utils__/factories/localFieldFactory.js index c13955cf0..3fe6197c9 100644 --- a/__utils__/factories/localFieldFactory.js +++ b/__utils__/factories/localFieldFactory.js @@ -2,18 +2,40 @@ const localField = require('../../server/utils/localField') let sequence = 0 -function localFieldFactory (prefix, { - en = `${prefix} en ${sequence++}`, - local = `${prefix} local ${sequence++}`, - lang = 'xx' -} = {}) { +async function localFieldFactory ( + api, + type, + prefix, + { + en = `${prefix} en ${sequence++}`, + local = `${prefix} local ${sequence++}`, + lang = 'xx', + apiInsertFormat = false + } = {} +) { const field = localField(prefix) - return { - [field.fieldNames.en]: en, - [field.fieldNames.local]: local, - [field.fieldNames.lang]: lang + if (api && type) { + await api.models.nomenclature.create({ + type, + labelEn: en + }) } + + return apiInsertFormat + ? { + [prefix]: { + label: { + en, + local + } + } + } + : { + [field.fieldNames.en]: en, + [field.fieldNames.local]: local, + [field.fieldNames.lang]: lang + } } module.exports = localFieldFactory diff --git a/__utils__/factories/settlementFactory.js b/__utils__/factories/settlementFactory.js index 86577919e..f6e0f5d30 100644 --- a/__utils__/factories/settlementFactory.js +++ b/__utils__/factories/settlementFactory.js @@ -2,13 +2,14 @@ const localFieldFactory = require('./localFieldFactory') let sequence = 0 async function settlementFactory (api, propOverrides, { - create = true + create = true, + apiInsertFormat = false } = {}) { sequence++ const record = { longitude: (sequence % 3600) / 10 - 180, latitude: sequence / 36000, - ...localFieldFactory('name'), + ...await localFieldFactory(api, null, 'name', { apiInsertFormat }), ...propOverrides } diff --git a/server/actions/filestorage.js b/server/actions/filestorage.js index de232c2a5..98ce48a4b 100644 --- a/server/actions/filestorage.js +++ b/server/actions/filestorage.js @@ -15,7 +15,7 @@ exports.uploader = upgradeAction('ah17', { }, function (err, id, stat) { if (err) return next(err) api.log('saved as #' + id, 'info', stat) - data.response.data = { id: id } + data.response.data = { id } next() }) } diff --git a/server/actions/zones.js b/server/actions/zones.js index 1c7c7072d..6392f7371 100644 --- a/server/actions/zones.js +++ b/server/actions/zones.js @@ -105,8 +105,8 @@ exports.zoneView = upgradeAction('ah17', { return zone }).then(function (zone) { switch (data.connection.extension) { - case 'gpx': return api.template.render('/zone.gpx.ejs', { zone: zone }) - case 'kml': return api.template.render('/zone.kml.ejs', { zone: zone }) + case 'gpx': return api.template.render('/zone.gpx.ejs', { zone }) + case 'kml': return api.template.render('/zone.kml.ejs', { zone }) default: return { data: zone.apiData(api) } } diff --git a/server/config/errors.js b/server/config/errors.js index 20c1b3974..0f20f0275 100644 --- a/server/config/errors.js +++ b/server/config/errors.js @@ -84,8 +84,8 @@ exports.default = { dataLengthTooLarge: function (maxLength, receivedLength) { return api.i18n.localize(['actionhero.errors.dataLengthTooLarge', { - maxLength: maxLength, - receivedLength: receivedLength + maxLength, + receivedLength }]) }, @@ -114,11 +114,11 @@ exports.default = { // /////////////// verbNotFound: function (connection, verb) { - return connection.localize(['actionhero.errors.verbNotFound', { verb: verb }]) + return connection.localize(['actionhero.errors.verbNotFound', { verb }]) }, verbNotAllowed: function (connection, verb) { - return connection.localize(['actionhero.errors.verbNotAllowed', { verb: verb }]) + return connection.localize(['actionhero.errors.verbNotAllowed', { verb }]) }, connectionRoomAndMessage: function (connection) { @@ -126,11 +126,11 @@ exports.default = { }, connectionNotInRoom: function (connection, room) { - return connection.localize(['actionhero.errors.connectionNotInRoom', { room: room }]) + return connection.localize(['actionhero.errors.connectionNotInRoom', { room }]) }, connectionAlreadyInRoom: function (connection, room) { - return connection.localize(['actionhero.errors.connectionAlreadyInRoom', { room: room }]) + return connection.localize(['actionhero.errors.connectionAlreadyInRoom', { room }]) }, connectionRoomHasBeenDeleted: function (room) { diff --git a/server/config/redis.js b/server/config/redis.js index dc0705a80..a24527f6d 100644 --- a/server/config/redis.js +++ b/server/config/redis.js @@ -46,17 +46,17 @@ exports.default = { _toExpand: false, client: { konstructor: require('ioredis'), - args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }], + args: [{ port, host, password, db, retryStrategy }], buildNew: true }, subscriber: { konstructor: require('ioredis'), - args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }], + args: [{ port, host, password, db, retryStrategy }], buildNew: true }, tasks: { konstructor: require('ioredis'), - args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }], + args: [{ port, host, password, db, retryStrategy }], buildNew: true } } diff --git a/server/helpers/incremental.js b/server/helpers/incremental.js index c11cc813f..8a02877ae 100644 --- a/server/helpers/incremental.js +++ b/server/helpers/incremental.js @@ -3,9 +3,9 @@ const inputHelpers = require('./inputs') const links = require('./links') module.exports = { - declareInputs: declareInputs, - prepareQuery: prepareQuery, - generateMeta: generateMeta + declareInputs, + prepareQuery, + generateMeta } /** diff --git a/server/helpers/links.js b/server/helpers/links.js index 16d8be95f..6ae91826f 100644 --- a/server/helpers/links.js +++ b/server/helpers/links.js @@ -2,8 +2,8 @@ const _ = require('lodash') const url = require('url') module.exports = { - fixParsedURL: fixParsedURL, - generateSelfUrl: generateSelfUrl + fixParsedURL, + generateSelfUrl } function fixParsedURL (api, data) { @@ -12,9 +12,9 @@ function fixParsedURL (api, data) { ? req.headers.host : 'localhost' const baseUrl = 'http' + (api.config.servers.web.secure ? 's' : '') + '://' + host - // eslint-disable-next-line node/no-deprecated-api + // eslint-disable-next-line n/no-deprecated-api const resolvedUrl = url.resolve(baseUrl, req && req.url ? req.url : '/') - // eslint-disable-next-line node/no-deprecated-api + // eslint-disable-next-line n/no-deprecated-api data.connection.rawConnection.parsedURL = url.parse(resolvedUrl, true) return data.connection.rawConnection.parsedURL } @@ -22,6 +22,6 @@ function fixParsedURL (api, data) { function generateSelfUrl (data, query) { return url.format(_.extend({}, data.connection.rawConnection.parsedURL, { search: undefined, - query: query + query })) } diff --git a/server/helpers/paging.js b/server/helpers/paging.js index 637b9578d..6222e73e7 100644 --- a/server/helpers/paging.js +++ b/server/helpers/paging.js @@ -7,9 +7,9 @@ module.exports = { * Default limit to use when no other value is specified */ defaultLimit: 20, - declareInputs: declareInputs, - prepareQuery: prepareQuery, - generateMeta: generateMeta + declareInputs, + prepareQuery, + generateMeta } /** diff --git a/server/initializers/filestorage.js b/server/initializers/filestorage.js index 152362608..bf813b1d5 100644 --- a/server/initializers/filestorage.js +++ b/server/initializers/filestorage.js @@ -60,7 +60,7 @@ module.exports = upgradeInitializer('ah17', { length: wb.size, custom: extra, type: mime, - filters: filters + filters } wm.write(JSON.stringify(meta)) wm.end(function () { @@ -87,7 +87,7 @@ module.exports = upgradeInitializer('ah17', { rm.pipe(concat(function (data) { try { const meta = JSON.parse(data) - api.log('blob', 'debug', { id: id, meta: meta }) + api.log('blob', 'debug', { id, meta }) const inflator = api.filestorage.inflator(meta.type || 'application/octet-stream', meta.filters || {}, meta) const strm = self.storage.createReadStream(meta.blob).pipe(inflator) next(null, strm, meta) @@ -104,7 +104,7 @@ module.exports = upgradeInitializer('ah17', { rm.pipe(concat(function (data) { try { const meta = JSON.parse(data) - api.log('blob', 'debug', { id: id, meta: meta }) + api.log('blob', 'debug', { id, meta }) self.storage.remove(meta.blob, function (err, res) { if (err) return next(err) self.storage.remove(id, next) diff --git a/server/initializers/formActions.js b/server/initializers/formActions.js index b2bcbd8ab..503d9b2d3 100644 --- a/server/initializers/formActions.js +++ b/server/initializers/formActions.js @@ -161,68 +161,31 @@ function generateExportAction (form) { */ function generateImportAction (form) { return async function (api, data, next) { - const importResult = { - total: data.params.items.length, - processed: 0, - created: 0, - updated: 0, - errors: [] - } - try { - await api.sequelize.sequelize.transaction(async (t) => { - if ((!api.forms.userCanManage(data.session.user, form.modelName)) || !data.params.user) { - data.params.user = data.session.userId - } + let allowed = false + if (api.forms.userCanManage(data.session.user, form.modelName)) { + allowed = true + } else if (!data.params.user) { + // regular users can only import own data + data.params.user = data.session.userId + allowed = true + } else if (data.session.userId === data.params.user) { + // need to have specified own user + allowed = true + } - for (let i = 0; i < data.params.items.length; i++) { - try { - const itemData = { - ...data.params.items[i], - organization: data.session.user.organizationSlug, - user: data.params.user, - language: data.params.language - } - - let record = await api.models[form.modelName].build({}) - record = await record.importData(itemData) - - const hash = record.calculateHash() - api.log('looking for %s with hash %s', 'info', form.modelName, hash) - const existing = await api.models[form.modelName].findOne({ where: { hash }, transaction: t }) - if (existing) { - api.log('found %s with hash %s, updating', 'info', form.modelName, hash) - await existing.importData(itemData) - record = await existing.save({ transaction: t }) - importResult.updated++ - } else { - api.log('not found %s with hash %s, creating', 'info', form.modelName, hash) - record = await record.save({ transaction: t }) - importResult.created++ - } - } catch (error) { - api.log(error, 'error') - importResult.errors.push({ row: i + 1, error: error.message }) - if (!data.params.skipErrors) { - importResult.created = 0 - importResult.updated = 0 - throw error - } - } - - importResult.processed++ - } + if (!allowed) throw new Error(api.config.errors.sessionNoPermission(data.connection)) - data.response.success = importResult.errors.length === 0 || data.params.skipErrors - }) + data.response.success = await api.tasks.enqueue('form:import', { + params: data.params, + user: data.session.user, + formName: form.modelName + }, 'low') + next() } catch (error) { - api.log('Error from transaction function', 'error') - data.response.data = importResult + api.log(error, 'error') next(error) } - - data.response.data = importResult - next() } } diff --git a/server/initializers/forms.js b/server/initializers/forms.js index 04bd0e798..5045dcde2 100644 --- a/server/initializers/forms.js +++ b/server/initializers/forms.js @@ -346,7 +346,7 @@ function generateExportData (form) { } function generateApiUpdate (fields) { - return async function (data, language, role) { + return async function (data, language, role, { validateNomenclatures = false, nomenclatures = [], species = [] } = {}) { const modelName = this.constructor.tableName await this.constructor.runHooks('beforeApiUpdate', this, data) await runFieldHooks(fields, 'beforeApiUpdate', this, data, language) @@ -363,6 +363,16 @@ function generateApiUpdate (fields) { // localField works with {bg, en}, while the multi nomenclature use [{label: {bg, en}}, ...] const val = data[name] + if (validateNomenclatures) { + const fieldNomenclatures = nomenclatures[field.relation.filter?.type] || [] + const found = (_.isArray(val) ? val : [val]) + .filter(v => v?.label?.en) + .every(v => fieldNomenclatures.find(n => n.label?.en === v.label?.en)) + if (!found) { + throw new Error(`[${modelName}.${name}] Invalid value: ${data[name]?.label?.en}`) + } + } + // directly set if null or not array if (val == null || !Array.isArray(val)) { localField(name).update(this, val != null ? val.label : null, language) @@ -393,7 +403,19 @@ function generateApiUpdate (fields) { if (!val) { this[name] = null + return + } + + if (field.relation.model === 'species' && validateNomenclatures) { + const modelSpecies = species[modelName === 'FormThreats' ? data.class : field.relation.filter?.type] || [] + const found = (_.isArray(val) ? val : [val]) + .filter(v => v) + .every(v => modelSpecies.find(s => s.label?.la === v)) + if (!found) { + throw new Error(`[${modelName}.${name}] Invalid value: ${data[name]}`) + } } + if (!_.isArray(val)) val = [val] this[name] = _.reduce(val, (sum, v) => sum + (sum ? ' | ' : '') + v, '') @@ -411,6 +433,14 @@ function generateApiUpdate (fields) { case 'settlement': { if (!_.has(data, name)) return + if (field.relation.model === 'nomenclature' && validateNomenclatures && data[name]?.label?.en) { + const fieldNomenclatures = nomenclatures[field.relation.filter?.type] || [] + const found = fieldNomenclatures.find(n => n.label?.en === data[name]?.label?.en) + if (!found) { + throw new Error(`[${modelName}.${name}] Invalid value: ${data[name]?.label?.en}`) + } + } + localField(name).update(this, data[name] != null ? data[name].label : null, language) break } @@ -418,6 +448,14 @@ function generateApiUpdate (fields) { case 'species': { if (!_.has(data, name)) return + if (field.relation.model === 'species' && validateNomenclatures && data[name]) { + const modelSpecies = species[modelName === 'FormThreats' ? data.class : field.relation.filter?.type] || [] + const found = modelSpecies.find(s => s.label?.la === data[name]) + if (!found) { + throw new Error(`[${modelName}.${name}] Invalid value: ${data[name]}`) + } + } + this[name] = data[name] break } @@ -498,6 +536,14 @@ function generateApiUpdate (fields) { if (data[name] === true || data[name] === false) { this[name] = data[name] + } else if (data[name] === 'true') { + this[name] = true + } else if (data[name] === 'false') { + this[name] = false + } else if (data[name] === '1') { + this[name] = true + } else if (data[name] === '0') { + this[name] = false } else if (data[name] == null) { this[name] = null } else { @@ -517,40 +563,6 @@ function generateApiUpdate (fields) { } } -function generateImportData (form) { - const importSkipFields = [ - 'id', - 'pictures', - 'track', - 'moderatorReview', - 'startDate', - 'startTime', - 'endDate', - 'endTime', - 'observationDate', - 'observationTime' - ] - - return async function (data) { - _.forEach(data, (value, name) => { - if (value === '') return - if (importSkipFields.includes(name)) return - - this[name] = value - if (name.endsWith('Local') && data.language && data.language !== 'en') { - this[`${name.substring(0, name.length - 5)}Lang`] = data.language || null - } - }) - - this.startDateTime = data.startDateTime || moment(data.startDate + ' ' + data.startTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() - this.endDateTime = data.endDateTime || moment(data.endDate + ' ' + data.endTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() - this.observationDateTime = data.observationDateTime || moment(data.observationDate + ' ' + data.observationTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() - this.userId = data.userId || data.user - - return this - } -} - function formOptions (form) { return { freezeTableName: true, @@ -572,8 +584,7 @@ function formOptions (form) { calculateHash: generateCalcHash(form.fields), apiData: generateApiData(form.fields), apiUpdate: generateApiUpdate(form.fields), - exportData: generateExportData(form), - importData: generateImportData(form) + exportData: generateExportData(form) }, hooks: form.hooks, validate: form.validate diff --git a/server/initializers/html5.js b/server/initializers/html5.js index a1b568a0f..4467d3337 100644 --- a/server/initializers/html5.js +++ b/server/initializers/html5.js @@ -5,7 +5,7 @@ module.exports = upgradeInitializer('ah17', { initialize: function (api, next) { api.staticFile.get = (function (originalGet) { return function (connection, callback, counter) { - api.log('staticFile.get', 'info', { file: connection.params.file, counter: counter }) + api.log('staticFile.get', 'info', { file: connection.params.file, counter }) if (connection.params.file !== api.config.general.directoryFileType) { api.staticFile.sendFileNotFound = (function (originalSendFileNotFound) { return function (connection, errorMessage, callback) { diff --git a/server/initializers/sequelize.js b/server/initializers/sequelize.js index 72bdeeccc..bb010cf5a 100644 --- a/server/initializers/sequelize.js +++ b/server/initializers/sequelize.js @@ -129,7 +129,7 @@ module.exports = upgradeInitializer('ah17', { sequelize: sequelizeInstance, - umzug: umzug, + umzug, connect: function (next) { const dir = path.normalize(api.projectRoot + '/models') diff --git a/server/initializers/session.js b/server/initializers/session.js index 460fd48f1..6fa03c0f1 100644 --- a/server/initializers/session.js +++ b/server/initializers/session.js @@ -31,9 +31,9 @@ module.exports = upgradeInitializer('ah17', { const sessionData = { userId: user.id, - csrfToken: csrfToken, + csrfToken, sesionCreatedAt: new Date().getTime(), - user: user + user } user.update({ lastLoginAt: new Date() }).then(function () { diff --git a/server/seeders/20151112150348-zone-location-seed.js b/server/seeders/20151112150348-zone-location-seed.js index ac475a67a..15c2d7e08 100644 --- a/server/seeders/20151112150348-zone-location-seed.js +++ b/server/seeders/20151112150348-zone-location-seed.js @@ -71,7 +71,7 @@ module.exports = { }) .then(function (locationId) { return queryInterface.bulkUpdate('Zones', { - locationId: locationId + locationId }, { id: zoneId }) }) .then(function () { diff --git a/server/seeders/20151112181152-users-seed.js b/server/seeders/20151112181152-users-seed.js index f77c9ae3f..d5c9442d3 100644 --- a/server/seeders/20151112181152-users-seed.js +++ b/server/seeders/20151112181152-users-seed.js @@ -39,7 +39,7 @@ module.exports = { .rawSelect('Users', { attributes: ['id'], where: { - email: email + email } }, 'id') .then(function (id) { @@ -51,7 +51,7 @@ module.exports = { return queryInterface.rawSelect('Users', { attributes: ['id'], where: { - email: email + email } }, 'id') }) @@ -78,7 +78,7 @@ module.exports = { } inserts.push(findId({ - email: email, + email, passwordHash: 'imported hash', firstName: record['Име'] && record['Име'].trim(), lastName: record['Фамилия'] && record['Фамилия'].trim(), diff --git a/server/seeders/20151220144314-nomenclatures-seed.js b/server/seeders/20151220144314-nomenclatures-seed.js index aea576340..4a2eed80a 100644 --- a/server/seeders/20151220144314-nomenclatures-seed.js +++ b/server/seeders/20151220144314-nomenclatures-seed.js @@ -10,9 +10,9 @@ function makeNomenclature (type, labelEn, labelBg) { console.error("TYPE too long: '" + type + "'") } return { - type: type, - labelEn: labelEn, - labelBg: labelBg, + type, + labelEn, + labelBg, createdAt: new Date(), updatedAt: new Date() } diff --git a/server/seeders/20160317003054-cbm-seeder.js b/server/seeders/20160317003054-cbm-seeder.js index aa9b6be38..a661e1adb 100644 --- a/server/seeders/20160317003054-cbm-seeder.js +++ b/server/seeders/20160317003054-cbm-seeder.js @@ -203,7 +203,7 @@ module.exports = { return queryInterface.rawSelect('Users', { attributes: ['id'], where: { - email: email + email } }, 'id').then(function (id) { if (id != null) { diff --git a/server/seeders/20160821161734-locations-ver2-seed.js b/server/seeders/20160821161734-locations-ver2-seed.js index 679aa180c..0d81f3717 100644 --- a/server/seeders/20160821161734-locations-ver2-seed.js +++ b/server/seeders/20160821161734-locations-ver2-seed.js @@ -123,7 +123,7 @@ module.exports = { // update zone with the location id .then(function (locationId) { return queryInterface.bulkUpdate('Zones', { - locationId: locationId + locationId }, { id: zoneId }) }) diff --git a/server/tasks/formImport.js b/server/tasks/formImport.js new file mode 100644 index 000000000..8d7c34734 --- /dev/null +++ b/server/tasks/formImport.js @@ -0,0 +1,118 @@ +const { api, Task } = require('actionhero') +const { prepareImportData } = require('../utils/import') + +const prepareNomenclatureData = async () => { + const nomenclatures = await api.models.nomenclature.findAll() + const species = await api.models.species.findAll() + + return { + nomenclatures: (nomenclatures?.map(n => n.apiData()) || []).reduce((acc, n) => { + if (!acc[n.type]) { + acc[n.type] = [] + } + acc[n.type].push(n) + return acc + }, {}), + species: (species?.map(s => s.apiData()) || []).reduce((acc, s) => { + if (!acc[s.type]) { + acc[s.type] = [] + } + acc[s.type].push(s) + return acc + }, {}) + } +} +module.exports = class FormImport extends Task { + constructor () { + super() + this.name = 'form:import' + this.description = 'form:import' + this.frequency = 0 + this.queue = 'low' + this.middleware = [] + } + + async run ({ params, user, formName }) { + const importResult = { + total: params.items.length, + processed: 0, + created: 0, + updated: 0, + errors: [] + } + + try { + const form = api.forms[formName] + let userAllowed = false + if (api.forms.userCanManage(user, form.modelName)) { + userAllowed = true + } + + await api.sequelize.sequelize.transaction(async (t) => { + for (let i = 0; i < params.items.length; i++) { + try { + const itemData = prepareImportData( + params.items[i], + (userAllowed ? params.items[i]?.userId : user.id) || user.id, + params.language, + user.organizationSlug + ) + + const nomenclatureData = await prepareNomenclatureData() + + let record = await api.models[form.modelName].build({}) + record = await record.apiUpdate( + itemData, + params.language, + null, + { + validateNomenclatures: true, + nomenclatures: nomenclatureData.nomenclatures, + species: nomenclatureData.species + }) + + const hash = record.calculateHash() + + api.log('looking for %s with hash %s', 'info', form.modelName, hash) + const existing = await api.models[form.modelName].findOne({ where: { hash }, transaction: t }) + + if (existing) { + api.log('found %s with hash %s, updating', 'info', form.modelName, hash) + await existing.apiUpdate(itemData, params.language) + record = await existing.save({ transaction: t }) + importResult.updated++ + } else { + api.log('not found %s with hash %s, creating', 'info', form.modelName, hash) + record = await record.save({ transaction: t }) + importResult.created++ + } + } catch (error) { + api.log(error, 'error') + importResult.errors.push({ row: i + 1, error: error.message }) + if (!params.skipErrors) { + importResult.created = 0 + importResult.updated = 0 + throw error + } + } + + importResult.processed++ + } + + const success = importResult.errors.length === 0 || params.skipErrors + + api.log('Sending import email notification', 'notice', { user }) + const successEmail = await api.tasks.enqueue('mail:send', { + mail: { to: user.email, subject: 'Import ready' }, + template: 'import_ready', + locals: { importResult, user } + }, 'default') + + return { success, successEmail } + }) + } catch (error) { + api.log('Error from transaction function', 'error') + throw error + } + } +} diff --git a/server/templates/import_ready/html.ejs b/server/templates/import_ready/html.ejs new file mode 100644 index 000000000..00da6e8ce --- /dev/null +++ b/server/templates/import_ready/html.ejs @@ -0,0 +1,61 @@ +
+

Your import is ready.

+ +
+

Summary

+ +
+ + <% if (importResult.errors && importResult.errors.lenght > 0) { %> +

If you want to skip the rows with errors, check the "Ignore errors" when importing the data.

+ <% } %> + +
+ +
+ +
+

Вашето импортиране е готово.

+ +
+

Обобщение

+ +
+ + <% if (importResult.errors && importResult.errors.lenght > 0) { %> +

Ако желаете да импортирате данните, като пропуснете редовете с грешка, маркирайте "Игнорирай грешките" при импорта.

+ <% } %> + +
diff --git a/server/utils/import.js b/server/utils/import.js new file mode 100644 index 000000000..cf6d51903 --- /dev/null +++ b/server/utils/import.js @@ -0,0 +1,85 @@ +const _ = require('lodash') +const moment = require('moment/moment') +const { api } = require('actionhero') + +const prepareImportData = (data, userId, language, organization) => { + const importSkipFields = [ + 'id', + 'pictures', + 'track', + 'moderatorReview', + 'startDate', + 'startTime', + 'endDate', + 'endTime', + 'observationDate', + 'observationTime' + ] + + const importItem = { + userId, + language, + organization + } + + _.forEach(data, (value, name) => { + if (value === '') return + if (importSkipFields.includes(name)) return + // check if value is object + if (typeof value === 'object' && value?.label?.en.includes(' | ')) { + importItem[name] = [] + const labelValues = Object.entries(value.label).map(([key, langValues]) => { + return { language: key, value: langValues.split(' | ') } + }).reduce((acc, curr) => { + acc[curr.language] = curr.value + return acc + }, {}) + labelValues.en.forEach((labelValue, index) => { + importItem[name].push({ + ...value, + label: Object.keys(labelValues).reduce((acc, curr) => { + acc[curr] = labelValues[curr][index] + return acc + }, {}) + }) + }) + } else { + importItem[name] = value + } + }) + + importItem.observationDateTime = data.observationDateTime || moment(data.observationDate + ' ' + data.observationTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() + if (data.startDateTime) { + importItem.startDateTime = data.startDateTime || moment(data.startDate + ' ' + data.startTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() || importItem.observationDateTime + } else { + importItem.startDateTime = + (data.startDate && data.startTime) + ? moment(data.startDate + ' ' + data.startTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() + : importItem.observationDateTime + } + + if (data.endDateTime) { + importItem.endDateTime = data.endDateTime + } else { + importItem.endDateTime = + (data.endDate && data.endTime) + ? moment(data.endDate + ' ' + data.endTime, api.config.formats.date + ' ' + api.config.formats.time).tz(api.config.formats.tz).toDate() + : importItem.observationDateTime + } + + importItem.userId = userId + importItem.user = userId + importItem.monitoringCode = data.monitoringCode || generateMonitoringCode(importItem) + + return importItem +} + +const generateMonitoringCode = (data) => { + let date = data.observationDateTime || data.startDateTime + if (date && date.toJSON) { date = date.toJSON() } + return '!IMPORT-' + date +} + +module.exports = { + prepareImportData +}