Feature/improve import (#1113)

- Validate nomenclature field values
- Move to async task

__tests__/actions/forms/fields/birdsNewSpeciesModeratorReview.js

@@ -43,7 +43,7 @@ describe('birdsNewSpeciesModeratorReview', () => {
           await bgatlasSpeciesFactory(setup.api, cell, species)
 
           const { id } = await formFactory(setup.api, {
-            species: species,
+            species,
             user: user.email,
             ...getCenter(cell.coordinates())
           })
@@ -89,13 +89,13 @@ describe('birdsNewSpeciesModeratorReview', () => {
         const species = await speciesFactory(setup.api, 'birds')
 
         const { id: record1 } = await formFactory(setup.api, {
-          species: species,
+          species,
           user: user.email,
           ...getCenter(cell.coordinates())
         })
 
         const { id: record2 } = await formFactory(setup.api, {
-          species: species,
+          species,
           user: user.email,
           ...getCenter(cell.coordinates())
         })

__tests__/actions/import.js

@@ -2,6 +2,8 @@
 /* globals setup */
 
 const forms = require('../../__utils__/forms.js')
+const userFactory = require('../../__utils__/factories/userFactory')
+const _ = require('lodash')
 
 const formsWithGenerators = forms.map((form) => {
   return {
@@ -10,6 +12,7 @@ const formsWithGenerators = forms.map((form) => {
   }
 })
 
+// Temporary skip all tests
 describe('Import', () => {
   afterEach(() => {
     return Promise.all(forms.map(form => {
@@ -20,18 +23,29 @@ describe('Import', () => {
     }))
   })
 
-  setup.describeAsAuth((runAction) => {
-    test.each(formsWithGenerators.map(form => [form.name, form]))('imports %s records', async (name, form) => {
-      const records = []
-      for (let i = 0; i < 5; i++) {
-        const recordData = await form.generator(setup.api, { notes: 'from import' }, { create: false })
-        records.push(recordData)
-      }
-
-      return runAction(`${form.name}:import`, { items: records }).then(function (response) {
-        return setup.api.models[form.name].findAll({ where: { notes: 'from import' } }).then(function (items) {
-          expect(items.length).toBe(5)
-        })
+  const roles = ['user', 'admin']
+
+  roles.forEach((role) => {
+    setup[`describeAs${_.capitalize(role.toLowerCase())}`]((specs) => {
+      test.each(formsWithGenerators.map(form => [form.name, form]))('imports %s records', async (name, form) => {
+        const user = await userFactory(setup.api, { role })
+
+        const records = []
+        for (let i = 0; i < 5; i++) {
+          const recordData = await form.generator(setup.api, { notes: 'from import' }, { create: false, apiInsertFormat: true })
+          records.push(recordData)
+        }
+
+        await setup.api.tasks.tasks['form:import'].run(
+          {
+            params: { items: records, user: user.id, language: 'en' },
+            user,
+            formName: form.name
+          }
+        )
+
+        const expectedItems = await setup.api.models[form.name].findAll({ where: { notes: 'from import' } })
+        expect(expectedItems.length).toBe(5)
       })
     })
   })

__tests__/actions/session.js

@@ -91,7 +91,7 @@ describe('Action session', () => {
       conn = await setup.api.specHelper.Connection.createAsync()
       conn.params = {
         email: user.email,
-        password: password
+        password
       }
 
       csrfToken = await setup.runAction('session:create', conn)

__tests__/tasks/formExport.js

@@ -23,7 +23,7 @@ test.each([
     },
     formName,
     outputType: 'csv',
-    user: user
+    user
   })
 
   expect(api.tasks.enqueue).toHaveBeenCalledWith('mail:send', {

__utils__/factories/formBirdsFactory.js

@@ -9,14 +9,24 @@ async function formBirdsFactory (api, {
   countMax = count,
   ...otherProps
 }, {
-  create = true
+  create = true,
+  apiInsertFormat = false
 } = {}) {
   species = await species
+
   const record = {
     ...await formCommonFactory(api, otherProps),
     species: species.labelLa || species,
-    ...localFieldFactory('countUnit'),
-    ...localFieldFactory('typeUnit'),
+    ...await localFieldFactory(api, 'birds_count_units', 'countUnit',
+      {
+        apiInsertFormat
+      }
+    ),
+    ...await localFieldFactory(api, 'birds_count_type', 'typeUnit',
+      {
+        apiInsertFormat
+      }
+    ),
     count,
     countMin,
     countMax,

__utils__/factories/formBirdsMigrationsFactory.js

@@ -9,15 +9,16 @@ async function formBirdsMigrationsFactory (api, {
   migrationPoint = poisFactory(api, { type: 'birds_migration_point' }),
   ...otherProps
 } = {}, {
-  create = true
+  create = true,
+  apiInsertFormat = false
 } = {}) {
   species = await species
   migrationPoint = await migrationPoint
   const record = {
     ...await formCommonFactory(api, otherProps),
     species: species.labelLa || species,
     count,
-    ...localFieldFactory('migrationPoint', { en: migrationPoint.labelEn }),
+    ...await localFieldFactory(api, 'birds_migration_point', 'migrationPoint', { en: migrationPoint.labelEn, apiInsertFormat }),
     ...otherProps
   }
 

__utils__/factories/formCBMFactory.js

@@ -9,15 +9,17 @@ async function formCBMFactory (api, {
   zone = zoneFactory(api),
   ...otherProps
 } = {}, {
-  create = true
+  create = true,
+  apiInsertFormat = false
 } = {}) {
   species = await species
   zone = await zone
   const record = {
     ...await formCommonFactory(api, otherProps),
-    ...localFieldFactory('distance'),
+    ...await localFieldFactory(api, 'cbm_distance', 'distance', { apiInsertFormat }),
     species: species.labelLa || species,
     count,
+    ...(apiInsertFormat ? { zone: zone.id } : { zoneId: zone.id }),
     zoneId: zone.id,
     ...otherProps
   }

__utils__/factories/formHerptilesFactory.js

@@ -2,7 +2,7 @@ const formCommonFactory = require('./formCommonFactory')
 const speciesFactory = require('./speciesFactory')
 
 async function formHerptilesFactory (api, {
-  species = speciesFactory(api, 'herptiles'),
+  species = speciesFactory(api, 'herptiles_name'),
   count = 1,
   ...otherProps
 } = {}, {

__utils__/factories/formInvertebratesFactory.js

@@ -2,7 +2,7 @@ const formCommonFactory = require('./formCommonFactory')
 const speciesFactory = require('./speciesFactory')
 
 async function formInvertebratesFactory (api, {
-  species = speciesFactory(api, 'invertebrates'),
+  species = speciesFactory(api, 'invertebrates_name'),
   count = 1,
   ...otherProps
 } = {}, {

__utils__/factories/formThreatsFactory.js

@@ -8,15 +8,17 @@ async function formThreatsFactory (api, {
   primaryType = 'threat',
   ...otherProps
 } = {}, {
-  create = true
+  create = true,
+  apiInsertFormat = false
 } = {}) {
   species = await species
   const record = {
     ...await formCommonFactory(api, otherProps),
     species: species.labelLa || species,
+    class: 'mammals',
     count,
     primaryType,
-    ...localFieldFactory('category'),
+    ...await localFieldFactory(api, 'threats_category', 'category', { apiInsertFormat }),
     ...otherProps
   }
 

__utils__/factories/localFieldFactory.js

@@ -2,18 +2,40 @@ const localField = require('../../server/utils/localField')
 
 let sequence = 0
 
-function localFieldFactory (prefix, {
-  en = `${prefix} en ${sequence++}`,
-  local = `${prefix} local ${sequence++}`,
-  lang = 'xx'
-} = {}) {
+async function localFieldFactory (
+  api,
+  type,
+  prefix,
+  {
+    en = `${prefix} en ${sequence++}`,
+    local = `${prefix} local ${sequence++}`,
+    lang = 'xx',
+    apiInsertFormat = false
+  } = {}
+) {
   const field = localField(prefix)
 
-  return {
-    [field.fieldNames.en]: en,
-    [field.fieldNames.local]: local,
-    [field.fieldNames.lang]: lang
+  if (api && type) {
+    await api.models.nomenclature.create({
+      type,
+      labelEn: en
+    })
   }
+
+  return apiInsertFormat
+    ? {
+        [prefix]: {
+          label: {
+            en,
+            local
+          }
+        }
+      }
+    : {
+        [field.fieldNames.en]: en,
+        [field.fieldNames.local]: local,
+        [field.fieldNames.lang]: lang
+      }
 }
 
 module.exports = localFieldFactory

__utils__/factories/settlementFactory.js

@@ -2,13 +2,14 @@ const localFieldFactory = require('./localFieldFactory')
 let sequence = 0
 
 async function settlementFactory (api, propOverrides, {
-  create = true
+  create = true,
+  apiInsertFormat = false
 } = {}) {
   sequence++
   const record = {
     longitude: (sequence % 3600) / 10 - 180,
     latitude: sequence / 36000,
-    ...localFieldFactory('name'),
+    ...await localFieldFactory(api, null, 'name', { apiInsertFormat }),
     ...propOverrides
   }
 

server/actions/filestorage.js

@@ -15,7 +15,7 @@ exports.uploader = upgradeAction('ah17', {
     }, function (err, id, stat) {
       if (err) return next(err)
       api.log('saved as #' + id, 'info', stat)
-      data.response.data = { id: id }
+      data.response.data = { id }
       next()
     })
   }

server/actions/zones.js

@@ -105,8 +105,8 @@ exports.zoneView = upgradeAction('ah17', {
       return zone
     }).then(function (zone) {
       switch (data.connection.extension) {
-        case 'gpx': return api.template.render('/zone.gpx.ejs', { zone: zone })
-        case 'kml': return api.template.render('/zone.kml.ejs', { zone: zone })
+        case 'gpx': return api.template.render('/zone.gpx.ejs', { zone })
+        case 'kml': return api.template.render('/zone.kml.ejs', { zone })
         default:
           return { data: zone.apiData(api) }
       }

server/config/errors.js

@@ -84,8 +84,8 @@ exports.default = {
 
       dataLengthTooLarge: function (maxLength, receivedLength) {
         return api.i18n.localize(['actionhero.errors.dataLengthTooLarge', {
-          maxLength: maxLength,
-          receivedLength: receivedLength
+          maxLength,
+          receivedLength
         }])
       },
 
@@ -114,23 +114,23 @@ exports.default = {
       // ///////////////
 
       verbNotFound: function (connection, verb) {
-        return connection.localize(['actionhero.errors.verbNotFound', { verb: verb }])
+        return connection.localize(['actionhero.errors.verbNotFound', { verb }])
       },
 
       verbNotAllowed: function (connection, verb) {
-        return connection.localize(['actionhero.errors.verbNotAllowed', { verb: verb }])
+        return connection.localize(['actionhero.errors.verbNotAllowed', { verb }])
       },
 
       connectionRoomAndMessage: function (connection) {
         return connection.localize('actionhero.errors.connectionRoomAndMessage')
       },
 
       connectionNotInRoom: function (connection, room) {
-        return connection.localize(['actionhero.errors.connectionNotInRoom', { room: room }])
+        return connection.localize(['actionhero.errors.connectionNotInRoom', { room }])
       },
 
       connectionAlreadyInRoom: function (connection, room) {
-        return connection.localize(['actionhero.errors.connectionAlreadyInRoom', { room: room }])
+        return connection.localize(['actionhero.errors.connectionAlreadyInRoom', { room }])
       },
 
       connectionRoomHasBeenDeleted: function (room) {

server/config/redis.js

@@ -46,17 +46,17 @@ exports.default = {
       _toExpand: false,
       client: {
         konstructor: require('ioredis'),
-        args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }],
+        args: [{ port, host, password, db, retryStrategy }],
         buildNew: true
       },
       subscriber: {
         konstructor: require('ioredis'),
-        args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }],
+        args: [{ port, host, password, db, retryStrategy }],
         buildNew: true
       },
       tasks: {
         konstructor: require('ioredis'),
-        args: [{ port: port, host: host, password: password, db: db, retryStrategy: retryStrategy }],
+        args: [{ port, host, password, db, retryStrategy }],
         buildNew: true
       }
     }

server/helpers/incremental.js

@@ -3,9 +3,9 @@ const inputHelpers = require('./inputs')
 const links = require('./links')
 
 module.exports = {
-  declareInputs: declareInputs,
-  prepareQuery: prepareQuery,
-  generateMeta: generateMeta
+  declareInputs,
+  prepareQuery,
+  generateMeta
 }
 
 /**

server/helpers/links.js

@@ -2,8 +2,8 @@ const _ = require('lodash')
 const url = require('url')
 
 module.exports = {
-  fixParsedURL: fixParsedURL,
-  generateSelfUrl: generateSelfUrl
+  fixParsedURL,
+  generateSelfUrl
 }
 
 function fixParsedURL (api, data) {
@@ -12,16 +12,16 @@ function fixParsedURL (api, data) {
     ? req.headers.host
     : 'localhost'
   const baseUrl = 'http' + (api.config.servers.web.secure ? 's' : '') + '://' + host
-  // eslint-disable-next-line node/no-deprecated-api
+  // eslint-disable-next-line n/no-deprecated-api
   const resolvedUrl = url.resolve(baseUrl, req && req.url ? req.url : '/')
-  // eslint-disable-next-line node/no-deprecated-api
+  // eslint-disable-next-line n/no-deprecated-api
   data.connection.rawConnection.parsedURL = url.parse(resolvedUrl, true)
   return data.connection.rawConnection.parsedURL
 }
 
 function generateSelfUrl (data, query) {
   return url.format(_.extend({}, data.connection.rawConnection.parsedURL, {
     search: undefined,
-    query: query
+    query
   }))
 }