diff --git a/src/api/routes.ts b/src/api/routes.ts index 5250e95b..13935781 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -168,12 +168,18 @@ api.get( // export current versions of all records in this notebook as csv api.get( - '/notebooks/:id/:viewid.csv', + '/notebooks/:id/:viewID.csv', requireAuthenticationAPI, async (req, res) => { if (req.user && userHasPermission(req.user, req.params.id, 'read')) { - res.setHeader('Content-Type', 'text/csv'); - streamNotebookRecordsAsCSV(req.params.id, req.params.viewid, res); + try { + res.setHeader('Content-Type', 'text/csv'); + streamNotebookRecordsAsCSV(req.params.id, req.params.viewID, res); + } catch (err) { + console.log('Error streaming CSV', err); + res.json({error: 'error creating CSV'}); + res.status(500).end(); + } } else { res.json({error: 'notebook not found'}); res.status(404).end(); diff --git a/src/buildconfig.ts b/src/buildconfig.ts index fa9882ea..edfce9d9 100644 --- a/src/buildconfig.ts +++ b/src/buildconfig.ts @@ -88,7 +88,7 @@ function is_testing() { function couchdb_internal_url(): string { let couchdb = process.env.COUCHDB_INTERNAL_URL; - const couchdbDefault = 'http://localhost:5984/'; + const couchdbDefault = 'http://localhost:5984'; if (couchdb === '' || couchdb === undefined) { console.log('COUCHDB_INTERNAL_URL not set, using default'); return couchdbDefault; @@ -103,7 +103,7 @@ function couchdb_internal_url(): string { function couchdb_public_url(): string { let couchdb = process.env.COUCHDB_PUBLIC_URL; - const couchdbDefault = 'http://localhost:5984/'; + const couchdbDefault = 'http://localhost:5984'; if (couchdb === '' || couchdb === undefined) { console.log('COUCHDB_PUBLIC_URL not set, using default'); return couchdbDefault; diff --git a/src/couchdb/index.ts b/src/couchdb/index.ts index 0d21cbb3..4655ced3 100644 --- a/src/couchdb/index.ts +++ b/src/couchdb/index.ts @@ -60,7 +60,7 @@ export const getDirectoryDB = (): PouchDB.Database | undefined => { if (!_directoryDB) { const pouch_options = pouchOptions(); - const directorydb = COUCHDB_INTERNAL_URL + DIRECTORY_DB_NAME; + const directorydb = COUCHDB_INTERNAL_URL + '/' + DIRECTORY_DB_NAME; try { _directoryDB = new PouchDB(directorydb, pouch_options); } catch (error) { @@ -82,7 +82,7 @@ export const getUsersDB = (): PouchDB.Database | undefined => { if (!_usersDB) { const pouch_options = pouchOptions(); - const dbName = COUCHDB_INTERNAL_URL + PEOPLE_DB_NAME; + const dbName = COUCHDB_INTERNAL_URL + '/' + PEOPLE_DB_NAME; _usersDB = new PouchDB(dbName, pouch_options); } @@ -92,7 +92,7 @@ export const getUsersDB = (): PouchDB.Database | undefined => { export const getProjectsDB = (): PouchDB.Database | undefined => { if (!_projectsDB) { const pouch_options = pouchOptions(); - const dbName = COUCHDB_INTERNAL_URL + PROJECTS_DB_NAME; + const dbName = COUCHDB_INTERNAL_URL + '/' + PROJECTS_DB_NAME; try { _projectsDB = new PouchDB(dbName, pouch_options); } catch (error) { @@ -105,7 +105,7 @@ export const getProjectsDB = (): PouchDB.Database | undefined => { export const getInvitesDB = (): PouchDB.Database | undefined => { if (!_invitesDB) { const pouch_options = pouchOptions(); - const dbName = COUCHDB_INTERNAL_URL + INVITE_DB_NAME; + const dbName = COUCHDB_INTERNAL_URL + '/' + INVITE_DB_NAME; try { _invitesDB = new PouchDB(dbName, pouch_options); } catch (error) { @@ -115,21 +115,6 @@ export const getInvitesDB = (): PouchDB.Database | undefined => { return _invitesDB; }; -export const createProjectDB = ( - dbName: string -): PouchDB.Database | undefined => { - const pouch_options = pouchOptions(); - - try { - const db = new PouchDB(COUCHDB_INTERNAL_URL + dbName, pouch_options); - return db; - } catch (error) { - console.error('error creating project database'); - console.error(error); - } - return undefined; -}; - export const getProjectMetaDB = async ( projectID: ProjectID ): Promise => { @@ -140,7 +125,8 @@ export const getProjectMetaDB = async ( projectID )) as unknown as ProjectObject; if (projectDoc.metadata_db) { - const dbname = COUCHDB_INTERNAL_URL + projectDoc.metadata_db.db_name; + const dbname = + COUCHDB_INTERNAL_URL + '/' + projectDoc.metadata_db.db_name; const pouch_options = pouchOptions(); if (LOCAL_COUCHDB_AUTH !== undefined) { @@ -166,7 +152,7 @@ export const getProjectDataDB = async ( projectID )) as unknown as ProjectObject; if (projectDoc.data_db) { - const dbname = COUCHDB_INTERNAL_URL + projectDoc.data_db.db_name; + const dbname = COUCHDB_INTERNAL_URL + '/' + projectDoc.data_db.db_name; const pouch_options = pouchOptions(); if (LOCAL_COUCHDB_AUTH !== undefined) { diff --git a/src/couchdb/notebooks.ts b/src/couchdb/notebooks.ts index 42709485..4a2c8847 100644 --- a/src/couchdb/notebooks.ts +++ b/src/couchdb/notebooks.ts @@ -19,15 +19,11 @@ */ import PouchDB from 'pouchdb'; -import { - createProjectDB, - getProjectDataDB, - getProjectMetaDB, - getProjectsDB, -} from '.'; +import {getProjectsDB} from '.'; import {CLUSTER_ADMIN_GROUP_NAME} from '../buildconfig'; import { ProjectID, + getProjectDB, resolve_project_id, notebookRecordIterator, } from 'faims3-datamodel'; @@ -244,12 +240,10 @@ export const createNotebook = async ( status: 'published', }; - // TODO: check whether the project database exists already... - const metaDB = createProjectDB(metaDBName); + const metaDB = await getProjectDB(project_id); if (!metaDB) { return undefined; } - // get roles from the notebook, ensure that 'user' and 'admin' are included const roles = metadata.accesses || ['admin', 'user', 'team']; if (roles.indexOf('user') < 0) { @@ -280,13 +274,11 @@ export const createNotebook = async ( // ensure that the name is in the metadata metadata.name = projectName.trim(); await writeProjectMetadata(metaDB, metadata); - // data database - const dataDB = createProjectDB(dataDBName); + const dataDB = await getDataDB(project_id); if (!dataDB) { return undefined; } - // can't save security on a memory database so skip this if we're testing if (process.env.NODE_ENV !== 'test') { const dataSecurity = await dataDB.security(); @@ -323,8 +315,8 @@ export const updateNotebook = async ( uispec: ProjectUIModel, metadata: any ) => { - const metaDB = await getProjectMetaDB(project_id); - const dataDB = await getProjectDataDB(project_id); + const metaDB = await getProjectDB(project_id); + const dataDB = await getDataDB(project_id); if (!dataDB || !metaDB) { return undefined; } @@ -399,8 +391,8 @@ export const deleteNotebook = async (project_id: string) => { if (projectsDB) { const projectDoc = await projectsDB.get(project_id); if (projectDoc) { - const metaDB = await getProjectMetaDB(project_id); - const dataDB = await getProjectDataDB(project_id); + const metaDB = await getProjectDB(project_id); + const dataDB = await getDataDB(project_id); if (metaDB && dataDB) { await metaDB.destroy(); await dataDB.destroy(); @@ -452,24 +444,31 @@ export const getNotebookMetadata = async ( project_id: string ): Promise => { const result: ProjectMetadata = {}; - try { - // get the metadata from the db - const projectDB = await getProjectMetaDB(project_id); - if (projectDB) { - const metaDocs = await projectDB.allDocs({include_docs: true}); - metaDocs.rows.forEach((doc: any) => { - const id: string = doc['id']; - if (id && id.startsWith(PROJECT_METADATA_PREFIX)) { - const key: string = id.substring(PROJECT_METADATA_PREFIX.length + 1); - result[key] = doc.doc.metadata; - } - }); - result.project_id = project_id; - return result; - } else { - console.error('no metadata database found for', project_id); + const isValid = await validateNotebookID(project_id); + if (isValid) { + try { + // get the metadata from the db + const projectDB = await getProjectDB(project_id); + if (projectDB) { + const metaDocs = await projectDB.allDocs({include_docs: true}); + metaDocs.rows.forEach((doc: any) => { + const id: string = doc['id']; + if (id && id.startsWith(PROJECT_METADATA_PREFIX)) { + const key: string = id.substring( + PROJECT_METADATA_PREFIX.length + 1 + ); + result[key] = doc.doc.metadata; + } + }); + result.project_id = project_id; + return result; + } else { + console.error('no metadata database found for', project_id); + } + } catch (error) { + console.log('unknown project', project_id); } - } catch (error) { + } else { console.log('unknown project', project_id); } return null; @@ -485,7 +484,7 @@ export const getNotebookUISpec = async ( ): Promise => { try { // get the metadata from the db - const projectDB = await getProjectMetaDB(project_id); + const projectDB = await getProjectDB(project_id); if (projectDB) { const uiSpec = (await projectDB.get('ui-specification')) as any; delete uiSpec._id; @@ -509,15 +508,17 @@ export const validateNotebookID = async ( project_id: string ): Promise => { try { - const projectDB = await getProjectMetaDB(project_id); - if (projectDB) { - return true; - } else { - return false; + const projectsDB = getProjectsDB(); + if (projectsDB) { + const projectDoc = await projectsDB.get(project_id); + if (projectDoc) { + return true; + } } } catch (error) { return false; } + return false; }; /** @@ -530,7 +531,6 @@ export const getNotebookRecords = async ( project_id: string ): Promise => { const records = await getRecordsWithRegex(project_id, '.*', true); - console.log(`got ${records.length} records from ${project_id}`); const fullRecords: any[] = []; for (let i = 0; i < records.length; i++) { const data = await getFullRecordData( @@ -557,51 +557,137 @@ const getRecordHRID = (record: any) => { * generate a suitable value for the CSV export from a field * value. Serialise filenames, gps coordinates, etc. */ -const csvFormatValue = (fieldName: string, value: any, hrid: string) => { +const csvFormatValue = ( + fieldName: string, + fieldType: string, + value: any, + hrid: string +) => { const result: {[key: string]: any} = {}; - if (value instanceof Array) { - if (value.length === 0) { - result[fieldName] = ''; - return result; - } - const valueList = value.map((v: any) => { - if (v instanceof File) { - return generateFilename(v, fieldName, hrid); - } else { - return v; + if (fieldType === 'faims-attachment::Files') { + if (value instanceof Array) { + if (value.length === 0) { + result[fieldName] = ''; + return result; } - }); - result[fieldName] = valueList.join(';'); + const valueList = value.map((v: any) => { + if (v instanceof File) { + return generateFilename(v, fieldName, hrid); + } else { + return v; + } + }); + result[fieldName] = valueList.join(';'); + } else { + result[fieldName] = value; + } return result; } // gps locations - if (value instanceof Object && 'geometry' in value) { - result[fieldName] = value; - result[fieldName + '_latitude'] = value.geometry.coordinates[0]; - result[fieldName + '_longitude'] = value.geometry.coordinates[1]; + if (fieldType === 'faims-pos::Location') { + if (value instanceof Object && 'geometry' in value) { + result[fieldName] = value; + result[fieldName + '_latitude'] = value.geometry.coordinates[0]; + result[fieldName + '_longitude'] = value.geometry.coordinates[1]; + } else { + result[fieldName] = value; + result[fieldName + '_latitude'] = ''; + result[fieldName + '_longitude'] = ''; + } return result; } + + if (fieldType === 'faims-core::Relationship') { + if (value instanceof Array) { + result[fieldName] = value + .map((v: any) => { + return `${v.relation_type_vocabPair[0]}/${v.record_id}`; + }) + .join(';'); + } else { + result[fieldName] = value; + } + return result; + } + // default to just the value result[fieldName] = value; return result; }; -const convertDataForOutput = (data: any, hrid: string) => { +const convertDataForOutput = ( + fields: {name: string; type: string}[], + data: any, + hrid: string +) => { let result: {[key: string]: any} = {}; - Object.keys(data).forEach((key: string) => { - const formattedValue = csvFormatValue(key, data[key], hrid); - result = {...result, ...formattedValue}; + fields.map((field: any) => { + if (field.name in data) { + const formattedValue = csvFormatValue( + field.name, + field.type, + data[field.name], + hrid + ); + result = {...result, ...formattedValue}; + } else { + console.error('field missing in data', field.name, data); + } }); return result; }; +export const getNotebookFields = async ( + project_id: ProjectID, + viewID: string +) => { + // work out what fields we're going to output from the uiSpec + const uiSpec = await getNotebookUISpec(project_id); + if (!uiSpec) { + throw new Error("can't find project " + project_id); + } + if (!(viewID in uiSpec.viewsets)) { + throw new Error(`invalid form ${viewID} not found in notebook`); + } + const views = uiSpec.viewsets[viewID].views; + const fields: string[] = []; + views.forEach((view: any) => { + uiSpec.fviews[view].fields.forEach((field: any) => { + fields.push(field); + }); + }); + return fields; +}; + +const getNotebookFieldTypes = async (project_id: ProjectID, viewID: string) => { + const uiSpec = await getNotebookUISpec(project_id); + if (!uiSpec) { + throw new Error("can't find project " + project_id); + } + if (!(viewID in uiSpec.viewsets)) { + throw new Error(`invalid form ${viewID} not found in notebook`); + } + const views = uiSpec.viewsets[viewID].views; + const fields: any[] = []; + views.forEach((view: any) => { + uiSpec.fviews[view].fields.forEach((field: any) => { + fields.push({ + name: field, + type: uiSpec.fields[field]['type-returned'], + }); + }); + }); + return fields; +}; + export const streamNotebookRecordsAsCSV = async ( project_id: ProjectID, viewID: string, res: NodeJS.WritableStream ) => { const iterator = await notebookRecordIterator(project_id, viewID); + const fields = await getNotebookFieldTypes(project_id, viewID); let stringifier: Stringifier | null = null; let {record, done} = await iterator.next(); @@ -616,7 +702,7 @@ export const streamNotebookRecordsAsCSV = async ( record.updated_by, record.updated.toISOString(), ]; - const outputData = convertDataForOutput(record.data, hrid); + const outputData = convertDataForOutput(fields, record.data, hrid); Object.keys(outputData).forEach((property: string) => { row.push(outputData[property]); }); @@ -676,7 +762,6 @@ export const streamNotebookFilesAsZip = async ( // all processed then we can finalize the archive archive.on('progress', (entries: any) => { if (!doneFinalize && allFilesAdded && entries.total === entries.processed) { - console.log('finalizing archive'); archive.finalize(); doneFinalize = true; } diff --git a/test/api.test.ts b/test/api.test.ts index 314ee855..d5aa516a 100644 --- a/test/api.test.ts +++ b/test/api.test.ts @@ -71,7 +71,6 @@ describe('API tests', () => { }); it('check is up - authenticated', async () => { - console.log('check is up - authenticated'); const result = await request(app) .get('/api/hello') .set('Authorization', `Bearer ${adminToken}`); @@ -205,7 +204,6 @@ describe('API tests', () => { }); it('can delete a notebook', async () => { - await resetDatabases(); const filename = 'notebooks/sample_notebook.json'; const jsonText = fs.readFileSync(filename, 'utf-8'); const {metadata, 'ui-specification': uiSpec} = JSON.parse(jsonText); diff --git a/test/couchdb.test.ts b/test/couchdb.test.ts index 2282d987..333c6c71 100644 --- a/test/couchdb.test.ts +++ b/test/couchdb.test.ts @@ -21,11 +21,7 @@ import PouchDB from 'pouchdb'; PouchDB.plugin(require('pouchdb-adapter-memory')); // enable memory adapter for testing PouchDB.plugin(require('pouchdb-find')); -import { - getDirectoryDB, - getProjectMetaDB, - initialiseDatabases, -} from '../src/couchdb'; +import {getDirectoryDB, initialiseDatabases} from '../src/couchdb'; import { createNotebook, getNotebookMetadata, @@ -45,9 +41,10 @@ import { userHasPermission, } from '../src/couchdb/users'; import {CONDUCTOR_INSTANCE_NAME} from '../src/buildconfig'; -import {ProjectUIModel} from 'faims3-datamodel'; +import {ProjectUIModel, getProjectDB} from 'faims3-datamodel'; import {expect} from 'chai'; import {resetDatabases} from './mocks'; +import {fail} from 'assert'; const uispec: ProjectUIModel = { fields: [], @@ -155,11 +152,15 @@ describe('notebook api', () => { const notebooks = await getNotebooks(user); expect(notebooks.length).to.equal(1); - const db = await getProjectMetaDB(projectID); + const db = await getProjectDB(projectID); if (db) { - const autoInc = (await db.get('local-autoincrementers')) as any; - expect(autoInc.references.length).to.equal(2); - expect(autoInc.references[0].form_id).to.equal('FORM1SECTION1'); + try { + const autoInc = (await db.get('local-autoincrementers')) as any; + expect(autoInc.references.length).to.equal(2); + expect(autoInc.references[0].form_id).to.equal('FORM1SECTION1'); + } catch (err) { + fail('could not get autoincrementers'); + } } } }); @@ -317,7 +318,7 @@ describe('notebook api', () => { const notebooks = await getNotebooks(user); expect(notebooks.length).to.equal(1); - const db = await getProjectMetaDB(projectID); + const db = await getProjectDB(projectID); if (db) { const newUISpec = await getNotebookUISpec(projectID); if (newUISpec) { @@ -330,7 +331,7 @@ describe('notebook api', () => { expect(newMetadata['name']).to.equal('Updated Test Notebook'); expect(newMetadata['project_lead']).to.equal('Bob Bobalooba'); } - const metaDB = await getProjectMetaDB(projectID); + const metaDB = await getProjectDB(projectID); if (metaDB) { const autoInc = (await metaDB.get('local-autoincrementers')) as any; expect(autoInc.references.length).to.equal(3); diff --git a/test/mocks.ts b/test/mocks.ts index 6b69649e..2c012513 100644 --- a/test/mocks.ts +++ b/test/mocks.ts @@ -3,12 +3,17 @@ import PouchDB from 'pouchdb'; PouchDB.plugin(require('pouchdb-adapter-memory')); // enable memory adapter for testing import {ProjectID, DBCallbackObject} from 'faims3-datamodel'; import {getProjectsDB, getUsersDB, initialiseDatabases} from '../src/couchdb'; +import {COUCHDB_INTERNAL_URL} from '../src/buildconfig'; const databaseList: any = {}; const getDatabase = async (databaseName: string) => { if (databaseList[databaseName] === undefined) { - const db = new PouchDB(databaseName, {adapter: 'memory'}); + // still use the COUCHDB URL setting to be consistent with + // other bits of the code, but this database will be in memory + const db = new PouchDB(COUCHDB_INTERNAL_URL + '/' + databaseName, { + adapter: 'memory', + }); databaseList[databaseName] = db; } return databaseList[databaseName]; @@ -56,7 +61,7 @@ export const cleanDataDBS = async () => { await db.destroy(); //await db.close(); } catch (err) { - console.error(err); + //console.error(err); } } }