redact fields in webapp. necessary refactoring (#470)

src/hooks/redaction.ts

@@ -137,3 +137,15 @@ export const publicApiTranscriptRedactionCondition: RedactCondition = (context,
     inPublicApi(context, redactable) && !bitmapsAlign(context, redactable, x => authBitmapExtractor(x, 'getTranscript'))
   )
 }
+
+const webappAuthBitmapExtractor = (redactable: Redactable, kind: keyof AuthorizationBitmapsDTO) => {
+  const actualKey = `bitmap${kind.charAt(0).toUpperCase() + kind.slice(1)}`
+  return redactable[actualKey] ?? BigInt(0)
+}
+
+export const webAppTranscriptRedactionCondition: RedactCondition = (context, redactable) => {
+  return (
+    !inPublicApi(context, redactable) &&
+    !bitmapsAlign(context, redactable, x => webappAuthBitmapExtractor(x, 'getTranscript'))
+  )
+}

src/hooks/results.js

@@ -1,35 +1,64 @@
-const debug = require('debug')('impresso/hooks:results');
+const debug = require('debug')('impresso/hooks:results')
+
+const asParamsWithUidFilter = (uids, extra) => {
+  return {
+    query: {
+      filters: [
+        {
+          type: 'uid',
+          q: uids,
+        },
+      ],
+      uids,
+      limit: uids.length,
+    },
+    ...extra,
+  }
+}
 
 /**
  * If a service is given, context.result.toBeResolved
  * will be filtered for serivce
  * @param  {[type]} service optional;
  * @return {[type]}         [description]
  */
-const resolve = service => async (context) => {
+const resolve = service => async context => {
   if (context.result.toBeResolved && context.result.toBeResolved.length) {
     if (service) {
-      context.result.toBeResolved = context.result.toBeResolved.filter(d => d.service === service);
+      context.result.toBeResolved = context.result.toBeResolved.filter(d => d.service === service)
     }
-    debug(`resolve: ${context.result.toBeResolved.length} services to resolve`, context.result.toBeResolved);
+    debug(`resolve: ${context.result.toBeResolved.length} services to resolve`, context.result.toBeResolved)
 
-    context.result.resolved = await Promise.all(context.result.toBeResolved.map(d => context.app.service(d.service).get(d.uids.join(','), {
-      authenticated: context.params.authenticated,
-      user: context.params.user,
-      findAll: true,
-      query: {
-        limit: d.uids.length,
-      },
-    }).then(results => ({
-      service: d.service,
-      data: results,
-    }))));
+    // context.result.resolved = await Promise.all(context.result.toBeResolved.map(d => context.app.service(d.service).get(d.uids.join(','), {
+    context.result.resolved = await Promise.all(
+      context.result.toBeResolved.map(d =>
+        context.app
+          .service(d.service)
+          // using `findInternal` to avoid hooks
+          .findInternal(
+            asParamsWithUidFilter(d.uids, {
+              authenticated: context.params.authenticated,
+              user: context.params.user,
+              findAll: true,
+              query: {
+                limit: d.uids.length,
+              },
+            })
+          )
+          .then(results => {
+            return {
+              service: d.service,
+              data: results.data,
+            }
+          })
+      )
+    )
     // delete context.result.toBeResolved;
   } else {
-    debug('resolve: result.toBeResolved, nothing to resolve.');
+    debug('resolve: result.toBeResolved, nothing to resolve.')
   }
-};
+}
 
 module.exports = {
   resolve,
-};
+}

src/middleware/transport.ts

@@ -1,28 +1,13 @@
 import type { Application as ExpressApplication } from '@feathersjs/express'
 import { json, rest, urlencoded } from '@feathersjs/express'
-import { Encoder, Decoder } from 'socket.io-parser'
+import { Decoder } from 'socket.io-parser'
 import cors from 'cors'
 import { ImpressoApplication } from '../types'
 // import { Server as EioWsServer } from 'eiows'
 
 import socketio from '@feathersjs/socketio'
 import { logger } from '../logger'
-
-/**
- * A replacer that encodes bigint as strings.
- */
-const customJSONReplacer = (key: string, value: any) => {
-  if (typeof value === 'bigint') {
-    return value.toString(10)
-  }
-  return value
-}
-
-class CustomEncoder extends Encoder {
-  constructor() {
-    super(customJSONReplacer)
-  }
-}
+import { CustomEncoder } from '../util/jsonEncoder'
 
 export default (app: ImpressoApplication & ExpressApplication) => {
   const isPublicApi = app.get('isPublicApi')

src/models/articles.model.js

@@ -723,9 +723,11 @@ class Article extends BaseArticle {
         locations: ArticleDPF.solrDPFsFactory(doc.loc_entities_dpfs),
         collections: doc.ucoll_ss,
         // permissions bitmaps
-        bitmapExplore: BigInt(doc.rights_bm_explore_l ?? 0),
-        bitmapGetTranscript: BigInt(doc.rights_bm_get_tr_l ?? 0),
-        bitmapGetImages: BigInt(doc.rights_bm_get_img_l ?? 0),
+        // if it's not defined, set max permissions for compatibility
+        // with old Solr version
+        bitmapExplore: BigInt(doc.rights_bm_explore_l ?? Number.MAX_SAFE_INTEGER),
+        bitmapGetTranscript: BigInt(doc.rights_bm_get_tr_l ?? Number.MAX_SAFE_INTEGER),
+        bitmapGetImages: BigInt(doc.rights_bm_get_img_l ?? Number.MAX_SAFE_INTEGER),
       })
 
       if (!doc.pp_plain) {

src/models/text-reuse-passages.model.js

@@ -103,8 +103,8 @@ class TextReusePassage {
     this.pageNumbers = pageNumbers
     this.collections = collections
 
-    this.bitmapExplore = bitmapExplore
-    this.bitmapGetTranscript = bitmapGetTranscript
+    this.bitmapExplore = BigInt(bitmapExplore ?? Number.MAX_SAFE_INTEGER)
+    this.bitmapGetTranscript = BigInt(bitmapGetTranscript ?? Number.MAX_SAFE_INTEGER)
   }
 
   static CreateFromSolr(fieldsToPropsMapper = SolrFieldsToPropsMapper) {

src/services/articles/articles.class.ts

@@ -162,31 +162,6 @@ export class Service {
   }
 
   async get(id: string, params: any) {
-    const uids = id.split(',')
-    if (uids.length > 1 || params.findAll) {
-      debug(
-        `[get] with ${uids.length} ids -> redirect to 'find', user:`,
-        params.user ? params.user.uid : 'no user found'
-      )
-
-      return this._find({
-        ...params,
-        findAll: true,
-        query: {
-          limit: 20,
-          filters: [
-            {
-              type: 'uid',
-              q: uids,
-            },
-          ],
-        },
-      }).then(res => res.data)
-    }
-    if (uids.length > 20) {
-      return []
-    }
-
     debug(`[get:${id}] with auth params:`, params.user ? params.user.uid : 'no user found')
     const fl = Article.ARTICLE_SOLR_FL_LIST_ITEM.concat([
       'lb_plain:[json]',

src/services/articles/articles.hooks.js

@@ -4,6 +4,7 @@ import {
   redactResponse,
   redactResponseDataItem,
   publicApiTranscriptRedactionCondition,
+  webAppTranscriptRedactionCondition,
   inPublicApi,
 } from '../../hooks/redaction'
 import { loadYamlFile } from '../../util/yaml'
@@ -27,6 +28,7 @@ const { obfuscate } = require('../../hooks/access-rights')
 const { SolrMappings } = require('../../data/constants')
 
 const contentItemRedactionPolicy = loadYamlFile(`${__dirname}/resources/contentItemRedactionPolicy.yml`)
+const contentItemRedactionPolicyWebApp = loadYamlFile(`${__dirname}/resources/contentItemRedactionPolicyWebApp.yml`)
 
 module.exports = {
   around: {
@@ -103,6 +105,7 @@ module.exports = {
       obfuscate(),
       transformResponseDataItem(transformContentItem, inPublicApi),
       redactResponseDataItem(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition),
+      redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition),
     ],
     get: [
       // save here cache, flush cache here
@@ -115,6 +118,7 @@ module.exports = {
       obfuscate(),
       transformResponse(transformContentItem, inPublicApi),
       redactResponse(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition),
+      redactResponse(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition),
     ],
     create: [],
     update: [],

src/services/articles/resources/contentItemRedactionPolicyWebApp.yml

@@ -0,0 +1,7 @@
+# yaml-language-server: $schema=../../../schema/common/redactionPolicy.json
+name: content-item-redaction-policy
+items:
+  - jsonPath: $.title
+    valueConverterName: redact
+  - jsonPath: $.excerpt
+    valueConverterName: redact

src/services/collections/collections.class.js

@@ -24,6 +24,14 @@ class Service {
   }
 
   async find(params) {
+    return this._find(params)
+  }
+
+  async findInternal(params) {
+    return this._find(params)
+  }
+
+  async _find(params) {
     const where = {
       [Op.not]: { status: Collection.STATUS_DELETED },
       [Op.and]: [
@@ -58,16 +66,6 @@ class Service {
   }
 
   async get(id, params) {
-    const uids = id.split(',')
-    if (params.findAll || (uids.length > 1 && uids.length < 20)) {
-      return this.find({
-        ...params,
-        query: {
-          ...params.query,
-          uids,
-        },
-      }).then(d => d.data)
-    }
     const where = {
       uid: id,
     }

src/services/collections/collections.service.js

@@ -23,6 +23,7 @@ module.exports = function (app) {
 
   // Get our initialized service so that we can register hooks
   const service = app.service('collections')
+  service.setup(app)
 
   service.hooks(hooks)
 }

src/services/jobs/jobs.class.js

@@ -10,7 +10,7 @@ class Service {
     this.options = options
   }
 
-  setup(app) {
+  setup(app, path) {
     this.app = app
     this.name = 'jobs'
     this.sequelizeService = new SequelizeService({

src/services/jobs/jobs.service.js

@@ -14,6 +14,7 @@ module.exports = function (app) {
 
   // Get our initialized service so that we can register hooks
   const service = app.service('jobs')
+  service.setup(app)
 
   service.hooks(hooks)
 }

src/services/me/me.service.js

@@ -1,19 +1,20 @@
 // Initializes the `me` service on path `/me`
-const createService = require('./me.class.js');
-const hooks = require('./me.hooks');
+const createService = require('./me.class.js')
+const hooks = require('./me.hooks')
 
 module.exports = function (app) {
-  const paginate = app.get('paginate');
+  const paginate = app.get('paginate')
 
   const options = {
     paginate,
-  };
+  }
 
   // Initialize our service with any options it requires
-  app.use('/me', createService(options));
+  app.use('/me', createService(options))
 
   // Get our initialized service so that we can register hooks
-  const service = app.service('me');
+  const service = app.service('me')
+  service.setup(app)
 
-  service.hooks(hooks);
-};
+  service.hooks(hooks)
+}

src/services/ngram-trends/ngram-trends.service.js

@@ -1,7 +1,9 @@
-const { NgramTrends } = require('./ngram-trends.class');
-const hooks = require('./ngram-trends.hooks');
+const { NgramTrends } = require('./ngram-trends.class')
+const hooks = require('./ngram-trends.hooks')
 
 module.exports = function (app) {
-  app.use('/ngram-trends', new NgramTrends());
-  app.service('ngram-trends').hooks(hooks);
-};
+  const service = new NgramTrends()
+  app.use('/ngram-trends', service)
+  app.service('ngram-trends').hooks(hooks)
+  service.setup(app)
+}

src/services/search-queries-comparison/search-queries-comparison.service.js

@@ -1,8 +1,10 @@
 // Initializes the `search-queries-comparison` service on path `/search-queries-comparison`
-const { SearchQueriesComparison } = require('./search-queries-comparison.class');
-const hooks = require('./search-queries-comparison.hooks');
+const { SearchQueriesComparison } = require('./search-queries-comparison.class')
+const hooks = require('./search-queries-comparison.hooks')
 
 module.exports = function (app) {
-  app.use('/search-queries-comparison', new SearchQueriesComparison());
-  app.service('search-queries-comparison').hooks(hooks);
-};
+  const service = new SearchQueriesComparison()
+  app.use('/search-queries-comparison', service)
+  app.service('search-queries-comparison').hooks(hooks)
+  service.setup(app)
+}

src/services/search/search.hooks.js

@@ -1,6 +1,11 @@
 import { authenticateAround as authenticate } from '../../hooks/authenticate'
 import { rateLimit } from '../../hooks/rateLimiter'
-import { redactResponseDataItem, inPublicApi, publicApiTranscriptRedactionCondition } from '../../hooks/redaction'
+import {
+  redactResponseDataItem,
+  inPublicApi,
+  publicApiTranscriptRedactionCondition,
+  webAppTranscriptRedactionCondition,
+} from '../../hooks/redaction'
 import { transformResponseDataItem, transformResponse, renameQueryParameters } from '../../hooks/transformation'
 import { transformBaseFind } from '../../transformers/base'
 import { transformContentItem } from '../../transformers/contentItem'
@@ -22,6 +27,9 @@ const { SolrMappings } = require('../../data/constants')
 const { SolrNamespaces } = require('../../solr')
 
 const contentItemRedactionPolicy = loadYamlFile(`${__dirname}/../articles/resources/contentItemRedactionPolicy.yml`)
+const contentItemRedactionPolicyWebApp = loadYamlFile(
+  `${__dirname}/../articles/resources/contentItemRedactionPolicyWebApp.yml`
+)
 
 const findQueryParamsRenamePolicy = {
   term: 'q',
@@ -112,6 +120,7 @@ module.exports = {
       protect('content'),
       transformResponseDataItem(transformContentItem, inPublicApi),
       redactResponseDataItem(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition),
+      redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition),
     ],
     get: [],
     create: [],

src/services/text-reuse-clusters/text-reuse-clusters.class.ts

@@ -45,8 +45,8 @@ function buildResponseClusters(
     ({ id, text: textSample, permissionBitmapExplore, permissionsBitmapGetTranscript }) => ({
       cluster: clustersById[id],
       textSample,
-      bitmapExplore: BigInt(permissionBitmapExplore ?? 0),
-      bitmapGetTranscript: BigInt(permissionsBitmapGetTranscript ?? 0),
+      bitmapExplore: BigInt(permissionBitmapExplore ?? Number.MAX_SAFE_INTEGER),
+      bitmapGetTranscript: BigInt(permissionsBitmapGetTranscript ?? Number.MAX_SAFE_INTEGER),
     })
   )
   return results