Merge pull request #15 from BonnierNews/clean-entity-history

Clean entity history

.gitignore

@@ -31,3 +31,5 @@ config/_revision
 
 # Atom CTAGS
 .tags
+
+*.log

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug current test file",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
+      "stopOnEntry": false,
+      "args": [
+        "${file}",
+        "--no-timeouts"
+      ],
+      "cwd": "${workspaceRoot}",
+      "runtimeExecutable": null,
+      "env": {
+        "NODE_ENV": "testing"
+      }
+    }
+  ]
+}

README.md

@@ -60,6 +60,29 @@ db.restore(versionId, (dbErr, entity) => {
 
 ```
 
+### Clean document version history
+```js
+
+const entity = {
+  id: "12903821",
+  type: "person", // type is required
+  attributes: {
+    name: "anonymous"
+  },
+  meta: {
+    correlationId: 456
+  }
+}
+
+db.cleanEntityHistory(entity, (dbErr) => {
+  if (dbErr) return dbErr;
+  // this will create a new version of the entity
+  // and remove all previous versions
+  // it can also be done on soft removed entities
+});
+
+```
+
 ### Get documents by relationships and externalIds
 #### Query by relationship
 ```js
@@ -123,6 +146,12 @@ version, this will create a new (latest) version of the document using
 the data from the specified version. This also marks the document as
 not removed.
 
+## Clean Entity History
+Will make a new version of the entity with the provided data and then remove all previous versions.
+Ignores if the entity is marked as removed or not.
+Should only be used when you are sure that you want to delete the entity version history,
+since it is not reversible. Good for gdpr!
+
 ## Adding extra tables
 
 Sometimes whats provided is not sufficient, to address this you could add your own tables to the db:
@@ -142,3 +171,4 @@ The files in the directory should be `.sql`-files and their name should start wi
 003-type-not-null.sql
 004-key-value.sql
 ```
+

lib/query.js

@@ -115,7 +115,7 @@ function remove(id, correlationId, cb) {
 
   load(id, (err, doc) => {
     if (err) return cb(err);
-    if (!doc) return cb(new Error("no such entity"));
+    if (!doc) return cb(new Error("No such entity"));
 
     const emptyDoc = {
       id: id,
@@ -237,10 +237,15 @@ function listVersions(entityId, force, cb) {
 }
 
 function upsert(entity, done) {
+  const force = false;
+  return upsertWithForce(entity, force, done);
+}
+
+function upsertWithForce(entity, force, done) {
   entity.id = entity.id || uuid.v4();
   let newVersionId;
   async.waterfall([
-    (cb) => insertVersion(entity, cb),
+    (cb) => insertVersion(entity, force, cb),
     (versionId, created, cb) => {
       if (versionId) {
         newVersionId = versionId;
@@ -275,26 +280,57 @@ function doUpsert(entity, versionId, created, done) {
   );
 }
 
-function insertVersion(entity, done) {
+function insertVersion(entity, force, done) {
   const versionId = uuid.v4();
   const correlationId = (entity.meta && entity.meta.correlationId) || null;
 
-  pgClient.query([
+  const q = [
     "INSERT into entity_version",
     "(version_id, entity_id, correlation_id, doc)",
-    "SELECT $1::text, $2::text, $3::text, $4::jsonb",
-    "WHERE NOT EXISTS (SELECT FROM entity WHERE entity_id = $2 AND entity_removed IS NOT NULL)",
-    "RETURNING created"
-  ].join(" "), [versionId, entity.id, correlationId, entity],
-  (err, res) => {
-    const responseHasTimestamp = (!err && res && res.rows && res.rows.length === 1);
-    const timestamp = responseHasTimestamp ? res.rows[0].created : undefined;
-
-    done(err, (res && res.rowCount > 0) ? versionId : null, timestamp);
+    "SELECT $1::text, $2::text, $3::text, $4::jsonb"
+  ];
+
+  if (!force) {
+    q.push("WHERE NOT EXISTS (SELECT FROM entity WHERE entity_id = $2 AND entity_removed IS NOT NULL)");
   }
+  q.push("RETURNING created");
+
+  pgClient.query(q.join(" "), [versionId, entity.id, correlationId, entity],
+    (err, res) => {
+      const responseHasTimestamp = (!err && res && res.rows && res.rows.length === 1);
+      const timestamp = responseHasTimestamp ? res.rows[0].created : undefined;
+
+      done(err, (res && res.rowCount > 0) ? versionId : null, timestamp);
+    }
   );
 }
 
+function cleanEntityHistory(entity, cb) {
+  if (!entity || !entity.id || !entity.type) {
+    return cb(new Error(`Missing required fields in entity: ${JSON.stringify(entity)}`));
+  }
+
+  load(entity.id, true, (err, doc) => {
+    if (err) return cb(err);
+    if (!doc) return cb(new Error("No such entity"));
+
+    return upsertWithForce(entity, true, (upsertErr, updatedEntity) => {
+      if (upsertErr) return cb(upsertErr);
+      removeEntityVersions(entity.id, updatedEntity.versionId, cb);
+    });
+  });
+}
+
+function removeEntityVersions(id, lastVersionId, cb) {
+  pgClient.query([
+    "DELETE FROM entity_version",
+    "WHERE entity_id = $1",
+    "AND version_id != $2"].join(" "), [id, lastVersionId], (err) => {
+    if (err) return cb(err);
+    return cb(null);
+  });
+}
+
 function getStatus(cb) {
   pgClient.query("select 1", [], (err) => {
     return cb(err);
@@ -312,5 +348,6 @@ module.exports = {
   restoreVersion,
   upsert,
   tables,
+  cleanEntityHistory,
   getStatus
 };

package.json

@@ -13,7 +13,7 @@
     "Markus Ekholm",
     "Ivan Malmberg"
   ],
-  "version": "2.3.2",
+  "version": "2.3.3",
   "scripts": {
     "test": "mocha",
     "posttest": "eslint --cache ."

test/feature/remove-feature.js

@@ -0,0 +1,183 @@
+"use strict";
+
+/* eslint no-undef: 0, new-cap: 0 */
+
+const crud = require("../../lib/query");
+const uuid = require("uuid");
+const helper = require("../../lib/testHelper");
+
+Feature("Clean version history for given entity", () => {
+  after(helper.tearDown);
+
+  const attributes = [
+    { name: "J Doe 1" },
+    { name: "J Doe 2" },
+    { name: "anonymous" },
+    undefined
+  ];
+
+  const entity = {
+    id: uuid.v4(),
+    type: "person"
+  };
+
+  const correlationIds = ["x", "y", "z"];
+  const expectedEntity = Object.assign({}, entity, {
+    attributes: attributes[2],
+    meta: {
+      correlationId: correlationIds[2]
+    }});
+
+  Scenario("Remove all previous versions of an entity", () => {
+    let entityVersions;
+
+    before((done) => {
+      helper.clearAndInit(done);
+    });
+
+    Given("a new entity is saved", (done) => {
+      entity.attributes = attributes[0];
+      entity.meta = { correlationId: correlationIds[0] };
+      crud.upsert(entity, done);
+    });
+
+    And("a second version is added to the entity", (done) => {
+      entity.attributes = attributes[1];
+      entity.meta = { correlationId: correlationIds[1] };
+      crud.upsert(entity, done);
+    });
+
+    When("cleaning the entity history", (done) => {
+      entity.attributes = attributes[2];
+      entity.meta = { correlationId: correlationIds[2] };
+      crud.cleanEntityHistory(entity, done);
+    });
+
+    And("we forcefully get all the versions", (done) => {
+      crud.listVersions(entity.id, true, (err, dbEntity) => {
+        if (err) return done(err);
+        entityVersions = dbEntity;
+        return done();
+      });
+    });
+
+    Then("there should only be one version", () => {
+      entityVersions.length.should.eql(1);
+    });
+
+    And("the entity should have the latest attributes", (done) => {
+      crud.load(entity.id, (err, anonymousEntity) => {
+        if (err) return done(err);
+        anonymousEntity.should.eql(expectedEntity);
+        return done();
+      });
+    });
+
+    And("the version should have the latest attributes", (done) => {
+      crud.loadVersion(entityVersions[0].versionId, (err, res) => {
+        if (err) return done(err);
+        res.entity.should.eql(expectedEntity);
+        res.correlationId.should.equal(correlationIds[2]);
+        return done();
+      });
+    });
+  });
+
+  Scenario("Remove all previous versions of an soft deleted entity", () => {
+    let entityVersions;
+
+    before((done) => {
+      helper.clearAndInit(done);
+    });
+
+    Given("a new entity is saved", (done) => {
+      entity.attributes = attributes[0];
+      entity.meta = { correlationId: correlationIds[0] };
+      crud.upsert(entity, done);
+    });
+
+    And("a second version is added to the entity", (done) => {
+      entity.attributes = attributes[1];
+      entity.meta = { correlationId: correlationIds[1] };
+      crud.upsert(entity, done);
+    });
+
+    And("we remove the entity", (done) => {
+      crud.remove(entity.id, done);
+    });
+
+    When("cleaning the entity history", (done) => {
+      entity.attributes = attributes[2];
+      entity.meta = { correlationId: correlationIds[2] };
+      crud.cleanEntityHistory(entity, done);
+    });
+
+    And("we forcefully get all the versions", (done) => {
+      crud.listVersions(entity.id, true, (err, dbEntity) => {
+        if (err) return done(err);
+        entityVersions = dbEntity;
+        return done();
+      });
+    });
+
+    Then("there should only be one version", () => {
+      entityVersions.length.should.eql(1);
+    });
+
+    And("the entity should have the latest attributes and only possible to fetch forcefully", (done) => {
+      crud.load(entity.id, true, (err, anonymousEntity) => {
+        if (err) return done(err);
+        anonymousEntity.should.eql(expectedEntity);
+        return done();
+      });
+    });
+
+    And("the version should have the latest attributes and only possible to fetch forcefully", (done) => {
+      crud.loadVersion(entityVersions[0].versionId, true, (err, res) => {
+        if (err) return done(err);
+        res.entity.should.eql(expectedEntity);
+        res.correlationId.should.equal(correlationIds[2]);
+        return done();
+      });
+    });
+  });
+
+  Scenario("Remove should only be possible for entities that exists", () => {
+    before((done) => {
+      helper.clearAndInit(done);
+    });
+
+    Given("there is no entity is saved", () => {});
+
+    When("cleaning the entity history of an entity that dose not exists", (done) => {
+      entity.attributes = attributes[2];
+      entity.meta = { correlationId: correlationIds[2] };
+      crud.cleanEntityHistory(entity, (err) => {
+        err.message.should.eql("No such entity");
+        return done();
+      });
+    });
+  });
+
+  Scenario("Entity needs to have required fields", () => {
+    Then("entity needs id", (done) => {
+      entity.id = null;
+      entity.attributes = attributes[2];
+      entity.meta = { correlationId: correlationIds[2] };
+      crud.cleanEntityHistory(entity, (err) => {
+        err.message.should.contain("Missing required fields in entity:");
+        return done();
+      });
+    });
+
+    And("entity needs type", (done) => {
+      entity.type = null;
+      entity.attributes = attributes[2];
+      entity.meta = { correlationId: correlationIds[2] };
+      crud.cleanEntityHistory(entity, (err) => {
+        err.message.should.contain("Missing required fields in entity:");
+        return done();
+      });
+    });
+  });
+});