diff --git a/.gitignore b/.gitignore index f208467a..f0777469 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,8 @@ lib-cov pids logs results -dump +dumps/* +!dumps/.gitkeep npm-debug.log node_modules diff --git a/docker-compose.yml b/docker-compose.yml index c0801065..b746535c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - "27017:27017" volumes: - mongo-data:/data/db + - ./dumps:/dumps networks: - api-db-network diff --git a/dumps/.gitkeep b/dumps/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/images/mongodb/Dockerfile b/images/mongodb/Dockerfile index 1a38d8a7..4d55a480 100644 --- a/images/mongodb/Dockerfile +++ b/images/mongodb/Dockerfile @@ -4,3 +4,4 @@ LABEL org.opencontainers.image.source https://github.com/sensebox/openSenseMap-A LABEL org.opencontainers.image.description "MongoDB development database for openSenseMap API" COPY ./osem_admin.sh /docker-entrypoint-initdb.d +COPY ./osem_restore-dumps.sh /docker-entrypoint-initdb.d diff --git a/images/mongodb/osem_restore-dumps.sh b/images/mongodb/osem_restore-dumps.sh new file mode 100644 index 00000000..c72e7d26 --- /dev/null +++ b/images/mongodb/osem_restore-dumps.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +USER=${OSEM_dbuser:-"admin"} +DATABASE=OSeM-api +PASS=${OSEM_dbuserpass:-"admin"} + +FILES="/dumps/*" +echo "Going to restore openSenseMap dumps from the path $FILES" +for f in $FILES +do + echo "Restoring $f dump..." + # take action on each file. $f store current file name + mongorestore --db OSeM-api --username $USER --password $PASS --authenticationDatabase OSeM-api --gzip --archive=$f + echo "Dump $f was restored" +done +echo "Finished restoring all dumps!" \ No newline at end of file diff --git a/packages/api/lib/controllers/boxesController.js b/packages/api/lib/controllers/boxesController.js index 73263ae3..932d3b4f 100644 --- a/packages/api/lib/controllers/boxesController.js +++ b/packages/api/lib/controllers/boxesController.js @@ -168,6 +168,7 @@ const updateBox = async function updateBox (req, res, next) { * ] */ const getBoxLocations = async function getBoxLocations (req, res, next) { + // TODO: do we really need this ?!?! -> just used in frontend to display the line / path try { const box = await Box.findBoxById(req._userParams.boxId, { onlyLocations: true, lean: false }); res.send(await box.getLocations(req._userParams)); diff --git a/packages/api/lib/routes.js b/packages/api/lib/routes.js index 702f3be3..0926de4f 100644 --- a/packages/api/lib/routes.js +++ b/packages/api/lib/routes.js @@ -82,7 +82,7 @@ const routes = { { path: `${boxesPath}/:boxId/sensors`, method: 'get', handler: measurementsController.getLatestMeasurements, reference: 'api-Measurements-getLatestMeasurements' }, { path: `${boxesPath}/:boxId/sensors/:sensorId`, method: 'get', handler: measurementsController.getLatestMeasurements, reference: 'api-Measurements-getLatestMeasurementOfSensor' }, { path: `${boxesPath}/:boxId/data/:sensorId`, method: 'get', handler: measurementsController.getData, reference: 'api-Measurements-getData' }, - { path: `${boxesPath}/:boxId/locations`, method: 'get', handler: boxesController.getBoxLocations, reference: 'api-Measurements-getLocations' }, + // { path: `${boxesPath}/:boxId/locations`, method: 'get', handler: boxesController.getBoxLocations, reference: 'api-Measurements-getLocations' }, // TODO: maybe we can use this later for tracks or somethings else { path: `${boxesPath}/data`, method: 'post', handler: measurementsController.getDataMulti, reference: 'api-Measurements-getDataMulti' }, { path: `${boxesPath}/:boxId/data`, method: 'post', handler: measurementsController.postNewMeasurements, reference: 'api-Measurements-postNewMeasurements' }, { path: `${boxesPath}/:boxId/:sensorId`, method: 'post', handler: measurementsController.postNewMeasurement, reference: 'api-Measurements-postNewMeasurement' }, diff --git a/packages/models/src/box/box.js b/packages/models/src/box/box.js index 1710f603..50bcdc6a 100644 --- a/packages/models/src/box/box.js +++ b/packages/models/src/box/box.js @@ -4,6 +4,7 @@ const { mongoose } = require('../db'), timestamp = require('mongoose-timestamp'), Schema = mongoose.Schema, { schema: sensorSchema, model: Sensor } = require('../sensor/sensor'), + { schema: locationSchema } = require('../location/location'), isEqual = require('lodash.isequal'), integrations = require('./integrations'), sensorLayouts = require('./sensorLayouts'), @@ -21,28 +22,6 @@ const { mongoose } = require('../db'), const templateSketcher = new Sketcher(); -const locationSchema = new Schema({ - type: { - type: String, - default: 'Point', - enum: ['Point'], // only 'Point' allowed - required: true - }, - coordinates: { - type: [Number], // lng, lat, [height] - required: true, - validate: [function validateCoordLength (c) { - return c.length === 2 || c.length === 3; - }, '{PATH} must have length 2 or 3'] - }, - timestamp: { - type: Date, - } -}, { - _id: false, - usePushEach: true -}); - //senseBox schema const boxSchema = new Schema({ name: { @@ -50,14 +29,18 @@ const boxSchema = new Schema({ required: true, trim: true }, - locations: { - type: [locationSchema], - required: true, - }, - currentLocation: { + location: { type: locationSchema, - required: true, + required: true }, + // locations: { + // type: [locationSchema], + // required: true, + // }, + // currentLocation: { + // type: locationSchema, + // required: true, + // }, exposure: { type: String, trim: true, @@ -279,6 +262,7 @@ boxSchema.statics.initNew = function ({ // create box document and persist in database return this.create({ name, + location, // new location field currentLocation: boxLocation, locations: [boxLocation], grouptag, @@ -405,61 +389,78 @@ boxSchema.methods.updateLocation = function updateLocation (coords, timestamp) { timestamp = utcNow(); } + const newLoc = { + type: 'Point', + coordinates: coords, + timestamp: timestamp, + }; + // search for temporally adjacent locations // (assuming that box.locations is ordered by location.timestamp) - let earlierLoc, laterLocIndex; - for (laterLocIndex = 0; laterLocIndex < box.locations.length; laterLocIndex++) { - earlierLoc = box.locations[laterLocIndex]; - if (!earlierLoc || timestamp.isBefore(earlierLoc.timestamp)) { - earlierLoc = box.locations[laterLocIndex - 1]; - break; - } - } + // let earlierLoc, laterLocIndex; + if (timestamp.isAfter(box.location.timestamp)) { - // check whether we insert a new location or update a existing one, depending on spatiotemporal setting - if (!earlierLoc && !coords) { - // the timestamp is earlier than any location we have, but no location is provided - // -> use the next laterLoc location (there is always one from registration) - box.locations[laterLocIndex].timestamp = timestamp; - - // update currentLocation when there's no later location - if (!box.locations[laterLocIndex + 1]) { - box.currentLocation = box.locations[laterLocIndex]; - } + box.location = newLoc; - return box.save().then(() => Promise.resolve(box.locations[laterLocIndex])); - } else if ( - !earlierLoc || - ( - coords && - !isEqual(earlierLoc.coordinates, coords) && - !timestamp.isSame(earlierLoc.timestamp) - ) - ) { - // insert a new location, if coords and timestamps differ from prevLoc - // (ensures that a box is not at multiple places at once), - // or there is no previous location - const newLoc = { - type: 'Point', - coordinates: coords, - timestamp: timestamp - }; + return box.save().then(() => Promise.resolve(newLoc)); + } - // insert the new location after earlierLoc in array - box.locations.splice(laterLocIndex, 0, newLoc); + return Promise.resolve(newLoc); - // update currentLocation when there's no later location - if (!box.locations[laterLocIndex + 1]) { - box.currentLocation = newLoc; - } + // for (laterLocIndex = 0; laterLocIndex < box.locations.length; laterLocIndex++) { + // earlierLoc = box.locations[laterLocIndex]; + // if (!earlierLoc || timestamp.isBefore(earlierLoc.timestamp)) { + // earlierLoc = box.locations[laterLocIndex - 1]; + // break; + // } + // } - return box.save() - .then(() => Promise.resolve(newLoc)); - } + // check whether we insert a new location or update a existing one, depending on spatiotemporal setting + // if (!earlierLoc && !coords) { + // // the timestamp is earlier than any location we have, but no location is provided + // // -> use the next laterLoc location (there is always one from registration) + // box.locations[laterLocIndex].timestamp = timestamp; + + // // update currentLocation when there's no later location + // if (!box.locations[laterLocIndex + 1]) { + // box.currentLocation = box.locations[laterLocIndex]; + // } + + // return box.save().then(() => Promise.resolve(box.locations[laterLocIndex])); + // } else if ( + // !earlierLoc || + // ( + // coords && + // !isEqual(earlierLoc.coordinates, coords) && + // !timestamp.isSame(earlierLoc.timestamp) + // ) + // ) { + // // insert a new location, if coords and timestamps differ from prevLoc + // // (ensures that a box is not at multiple places at once), + // // or there is no previous location + // const newLoc = { + // type: 'Point', + // coordinates: coords, + // timestamp: timestamp + // }; + + // // insert the new location after earlierLoc in array + // // box.locations.splice(laterLocIndex, 0, newLoc); + + // // update currentLocation when there's no later location + // if (!box.locations[laterLocIndex + 1]) { + // box.currentLocation = newLoc; + // } + + // return box.save() + // .then(() => Promise.resolve(newLoc)); + // } // coords and timestamps are equal or not provided // -> return unmodified previous location - return Promise.resolve(earlierLoc); + // return Promise.resolve(earlierLoc); + + // return box.save().then(() => Promise.resolve(newLoc)); }; boxSchema.methods.saveMeasurement = function saveMeasurement (measurement) { @@ -535,52 +536,56 @@ boxSchema.methods.saveMeasurementsArray = function saveMeasurementsArray (measur // iterate over all new measurements to check for location updates let m = 0; - const newLocations = []; + // const newLocations = []; + let latestNewLocation = box.location; // neuste location der messungen die geschickt wurden while (m < measurements.length) { // find the location in both new and existing locations, which is newest // in relation to the measurent time. (box.locations is sorted by date) - const earlierLocOld = findEarlierLoc(box.locations, measurements[m]), - earlierLocNew = findEarlierLoc(newLocations, measurements[m]); - - let loc = earlierLocOld; - if ( - earlierLocNew && - parseTimestamp(earlierLocOld.timestamp).isBefore(earlierLocNew.timestamp) - ) { - loc = earlierLocNew; - } + // const earlierLocOld = findEarlierLoc(box.locations, measurements[m]), + // earlierLocNew = findEarlierLoc(newLocations, measurements[m]); + + // let loc = earlierLocOld; + // if ( + // earlierLocNew && + // parseTimestamp(earlierLocOld.timestamp).isBefore(earlierLocNew.timestamp) + // ) { + // loc = earlierLocNew; + // } // if measurement is earlier than first location (only occurs in first iteration) // use the first location of the box and redate it - if (!loc) { - loc = box.locations[0]; - loc.timestamp = measurements[m].createdAt; - } + // if (!loc) { + // loc = box.locations[0]; + // loc.timestamp = measurements[m].createdAt; + // } // check if new location equals the found location. // if not create a new one, else reuse the found location - if ( - measurements[m].location && - !isEqual(loc.coordinates, measurements[m].location) - ) { - loc = { - type: 'Point', - coordinates: measurements[m].location, - timestamp: measurements[m].createdAt - }; - - newLocations.push(loc); + if (measurements[m].location) { + if (measurements[m].location.timestamp.isAfter(latestNewLocation.timestamp)) { + latestNewLocation = { + type: 'Point', + coordinates: measurements[m].location, + timestamp: measurements[m].createdAt, + }; + } } + m++; + // do { + // m++; + // } while (m < measurements.length); + + // TODO: wenn keine location dann im response box.location // apply location to all measurements with missing or equal location. - do { - measurements[m].location = { type: 'Point', coordinates: loc.coordinates }; - m++; - } while ( - m < measurements.length && - (!measurements[m].location || isEqual(measurements[m].location, loc.coordinates)) - ); + // do { + // measurements[m].location = { type: 'Point', coordinates: loc.coordinates }; + // m++; + // } while ( + // m < measurements.length && + // (!measurements[m].location || isEqual(measurements[m].location, loc.coordinates)) + // ); } // save new measurements @@ -625,20 +630,20 @@ boxSchema.methods.saveMeasurementsArray = function saveMeasurementsArray (measur updateQuery.$set['lastMeasurementAt'] = lastMeasurementAt; } - if (newLocations.length) { + if (latestNewLocation) { // add the new locations to the box - updateQuery.$push = { - locations: { $each: newLocations, $sort: { timestamp: 1 } } - }; + // updateQuery.$push = { + // locations: { $each: newLocations, $sort: { timestamp: 1 } } + // }; // update currentLocation if necessary - const latestNewLocation = newLocations[newLocations.length - 1]; - if (latestNewLocation.timestamp.isAfter(box.currentLocation.timestamp)) { + // const latestNewLocation = newLocations[newLocations.length - 1]; + if (latestNewLocation.timestamp.isAfter(box.location.timestamp)) { if (!updateQuery.$set) { updateQuery.$set = {}; } - updateQuery.$set.currentLocation = latestNewLocation; + updateQuery.$set.location = latestNewLocation; } } @@ -727,7 +732,7 @@ boxSchema.statics.findMeasurementsOfBoxesStream = function findMeasurementsOfBox // store all matching sensors under sensors[sensorId] for (let i = 0, len = boxData.length; i < len; i++) { for (let j = 0, sensorslen = boxData[i].sensors.length; j < sensorslen; j++) { - if (boxData[i].sensors[j][sensorProperty].toString() === phenomenon) { + if (boxData[i].sensors[j][sensorProperty] && boxData[i].sensors[j][sensorProperty].toString() === phenomenon) { const sensor = boxData[i].sensors[j]; sensor.lat = boxData[i].currentLocation.coordinates[1]; @@ -901,7 +906,7 @@ boxSchema.methods.updateBox = function updateBox (args) { // run location update logic, if a location was provided. const locPromise = location - ? box.updateLocation(location).then(loc => box.set({ currentLocation: loc })) + ? box.updateLocation(location).then(loc => box.set({ location: loc })) : Promise.resolve(); return locPromise.then(function () { diff --git a/packages/models/src/location/location.js b/packages/models/src/location/location.js new file mode 100644 index 00000000..89a2a7f2 --- /dev/null +++ b/packages/models/src/location/location.js @@ -0,0 +1,36 @@ +'use strict'; + +const { mongoose } = require('../db'), + Schema = mongoose.Schema; + +const locationSchema = new Schema( + { + type: { + type: String, + default: 'Point', + enum: ['Point'], // only 'Point' allowed + required: true, + }, + coordinates: { + type: [Number], // lng, lat, [height] + required: true, + validate: [ + function validateCoordLength (c) { + return c.length === 2 || c.length === 3; + }, + '{PATH} must have length 2 or 3', + ], + }, + timestamp: { + type: Date, + }, + }, + { + _id: false, + usePushEach: true, + } +); + +module.exports = { + schema: locationSchema +}; diff --git a/packages/models/src/measurement/measurement.js b/packages/models/src/measurement/measurement.js index bf8d7e04..fc209a1d 100644 --- a/packages/models/src/measurement/measurement.js +++ b/packages/models/src/measurement/measurement.js @@ -3,39 +3,32 @@ const { mongoose } = require('../db'), moment = require('moment'), decodeHandlers = require('./decoding'), - ModelError = require('../modelError'); + ModelError = require('../modelError'), + { schema: locationSchema } = require('../location/location'); -const measurementSchema = new mongoose.Schema({ - value: { - type: String, - required: true - }, - sensor_id: { - type: mongoose.Schema.Types.ObjectId, - ref: 'Sensor', - required: true - }, - createdAt: { - type: Date, - required: true, - default: moment.utc().toDate() - }, - location: { - type: { +const measurementSchema = new mongoose.Schema( + { + value: { type: String, - default: 'Point', - enum: ['Point'], // only 'Point' allowed - required: true + required: true, }, - coordinates: { - type: [Number], // lng, lat, [height] + sensor_id: { + type: mongoose.Schema.Types.ObjectId, + ref: 'Sensor', required: true, - validate: [function validateCoordLength (c) { - return c.length === 2 || c.length === 3; - }, '{PATH} has not length 2 or 3'] - } - } -}, { usePushEach: true }); + }, + createdAt: { + type: Date, + required: true, + default: moment.utc().toDate(), + }, + location: { + type: locationSchema, + required: false, + }, + }, + { usePushEach: true } +); measurementSchema.index({ sensor_id: 1, createdAt: -1 });