fixes inter service access and bigint serialization for socket.io

src/configuration.ts

@@ -2,6 +2,8 @@ import { default as feathersConfiguration } from '@feathersjs/configuration'
 import type { FromSchema, JSONSchemaDefinition } from '@feathersjs/schema'
 import { Ajv, getValidator } from '@feathersjs/schema'
 import type { RedisClientOptions } from 'redis'
+import { Cache } from 'cache-manager'
+
 import type { RateLimiterConfiguration } from './services/internal/rateLimiter/redis'
 import { Sequelize } from 'sequelize'
 import { CeleryClient } from './celery'
@@ -120,6 +122,7 @@ export interface Configuration {
   celeryClient?: CeleryClient
   media?: MediaConfiguration
   solr: SolrConfiguration
+  cacheManager: Cache
 
   impressoNerServiceUrl?: string
 

src/middleware/transport.ts

@@ -1,12 +1,29 @@
 import type { Application as ExpressApplication } from '@feathersjs/express'
 import { json, rest, urlencoded } from '@feathersjs/express'
+import { Encoder, Decoder } from 'socket.io-parser'
 import cors from 'cors'
 import { ImpressoApplication } from '../types'
 // import { Server as EioWsServer } from 'eiows'
 
 import socketio from '@feathersjs/socketio'
 import { logger } from '../logger'
 
+/**
+ * A replacer that encodes bigint as strings.
+ */
+const customJSONReplacer = (key: string, value: any) => {
+  if (typeof value === 'bigint') {
+    return value.toString(10)
+  }
+  return value
+}
+
+class CustomEncoder extends Encoder {
+  constructor() {
+    super(customJSONReplacer)
+  }
+}
+
 export default (app: ImpressoApplication & ExpressApplication) => {
   const isPublicApi = app.get('isPublicApi')
 
@@ -34,6 +51,8 @@ export default (app: ImpressoApplication & ExpressApplication) => {
             credentials: true,
             origin: app.get('allowedCorsOrigins') ?? [],
           },
+          parser: { Encoder: CustomEncoder, Decoder },
+          // parser: customParser,
           // wsEngine: EioWsServer,
         },
         io => {

src/services/articles/articles.class.js → src/services/articles/articles.class.ts

@@ -1,49 +1,89 @@
-const { keyBy } = require('lodash')
-const debug = require('debug')('impresso/services:articles')
-const { Op } = require('sequelize')
-const { NotFound } = require('@feathersjs/errors')
-
-const SequelizeService = require('../sequelize.service')
-const SolrService = require('../solr.service')
-const Article = require('../../models/articles.model')
-const Issue = require('../../models/issues.model')
-const { measureTime } = require('../../util/instruments')
-
-async function getIssues(request, app) {
+import { keyBy } from 'lodash'
+import Debug from 'debug'
+import { Op } from 'sequelize'
+import { NotFound } from '@feathersjs/errors'
+
+import initSequelizeService, { Service as SequelizeService } from '../sequelize.service'
+import initSolrService, { Service as SolrService } from '../solr.service'
+import Article from '../../models/articles.model'
+import Issue from '../../models/issues.model'
+import { measureTime } from '../../util/instruments'
+import { ImpressoApplication } from '../../types'
+import { SlimUser } from '../../authentication'
+
+const debug = Debug('impresso/services:articles')
+
+async function getIssues(request: Record<string, any>, app: ImpressoApplication) {
   const sequelize = app.get('sequelizeClient')
   const cacheManager = app.get('cacheManager')
-  const cacheKey = SequelizeService.getCacheKeyForReadSqlRequest(request, 'issues')
+  const cacheKey = initSequelizeService.getCacheKeyForReadSqlRequest(request, 'issues')
 
   return cacheManager
     .wrap(cacheKey, async () =>
       Issue.sequelize(sequelize)
         .findAll(request)
-        .then(rows => rows.map(d => d.get()))
+        .then((rows: any[]) => rows.map(d => d.get()))
     )
     .then(rows => keyBy(rows, 'uid'))
 }
 
-class Service {
-  constructor({ name = '', app = undefined } = {}) {
+interface ServiceOptions {
+  name?: string
+  app?: ImpressoApplication
+}
+
+interface FindOptions {
+  query: {
+    filters?: any[]
+
+    // things needed by SolService.find
+    sq?: string
+    sfq?: string
+    limit?: number
+    offset?: number
+    facets?: string[]
+    order_by?: string
+    highlight_by?: boolean
+    collapse_by?: string
+    collapse_fn?: string
+    requestOriginalPath?: boolean
+  }
+  user: SlimUser
+
+  // things needed by SolService.find
+  fl?: string[]
+}
+
+export class Service {
+  name: string
+  app?: ImpressoApplication
+  SequelizeService: SequelizeService
+  SolrService: SolrService
+
+  constructor({ name = '', app = undefined }: ServiceOptions = {}) {
     this.name = String(name)
     this.app = app
-    this.SequelizeService = SequelizeService({
+    this.SequelizeService = initSequelizeService({
       app,
       name,
       cacheReads: true,
     })
-    this.SolrService = SolrService({
+    this.SolrService = initSolrService({
       app,
       name,
       namespace: 'search',
     })
   }
 
-  async find(params) {
+  async find(params: any) {
     return await this._find(params)
   }
 
-  async _find(params) {
+  async findInternal(params: any) {
+    return await this._find(params)
+  }
+
+  async _find(params: FindOptions) {
     const fl = Article.ARTICLE_SOLR_FL_LIST_ITEM
     const pageUids = (params.query.filters || []).filter(d => d.type === 'page').map(d => d.q)
 
@@ -70,7 +110,7 @@ class Service {
           ...params,
           scope: 'get',
           where: {
-            uid: { [Op.in]: results.data.map(d => d.uid) },
+            uid: { [Op.in]: results.data.map((d: { uid: string }) => d.uid) },
           },
           limit: results.data.length,
           order_by: [['uid', 'DESC']],
@@ -87,16 +127,16 @@ class Service {
     const issuesRequest = {
       attributes: ['accessRights', 'uid'],
       where: {
-        uid: { [Op.in]: results.data.map(d => d.issue.uid) },
+        uid: { [Op.in]: results.data.map((d: any) => d.issue.uid) },
       },
     }
-    const getRelatedIssuesPromise = measureTime(() => getIssues(issuesRequest, this.app), 'articles.find.db.issues')
+    const getRelatedIssuesPromise = measureTime(() => getIssues(issuesRequest, this.app!), 'articles.find.db.issues')
 
     // do the loop
     return Promise.all([getAddonsPromise, getRelatedIssuesPromise]).then(([addonsIndex, issuesIndex]) => ({
       ...results,
-      data: results.data.map(article => {
-        if (issuesIndex[article.issue.uid]) {
+      data: results.data.map((article: Article) => {
+        if (article?.issue?.uid != null && issuesIndex[article?.issue?.uid]) {
           article.issue.accessRights = issuesIndex[article.issue.uid].accessRights
         }
         if (!addonsIndex[article.uid]) {
@@ -110,18 +150,18 @@ class Service {
           // it came from cache. Otherwise it is a model instance and it was
           // loaded from the database.
           // This should be moved to the SequelizeService layer.
-          article.pages = addonsIndex[article.uid].pages.map(d => (d.constructor === Object ? d : d.toJSON()))
+          article.pages = addonsIndex[article.uid].pages.map((d: any) => (d.constructor === Object ? d : d.toJSON()))
         }
         if (pageUids.length === 1) {
-          article.regions = article.regions.filter(r => pageUids.indexOf(r.pageUid) !== -1)
+          article.regions = article?.regions?.filter(r => pageUids.indexOf(r.pageUid) !== -1)
         }
         article.assignIIIF()
         return article
       }),
     }))
   }
 
-  async get(id, params) {
+  async get(id: string, params: any) {
     const uids = id.split(',')
     if (uids.length > 1 || params.findAll) {
       debug(
@@ -187,7 +227,7 @@ class Service {
       ),
       measureTime(
         () =>
-          Issue.sequelize(this.app.get('sequelizeClient')).findOne({
+          Issue.sequelize(this.app!.get('sequelizeClient')).findOne({
             attributes: ['accessRights'],
             where: {
               uid: id.split(/-i\d{4}/).shift(),
@@ -199,9 +239,9 @@ class Service {
       .then(([article, addons, issue]) => {
         if (addons) {
           if (issue) {
-            article.issue.accessRights = issue.accessRights
+            article.issue.accessRights = (issue as any).accessRights
           }
-          article.pages = addons.pages.map(d => d.toJSON())
+          article.pages = addons.pages.map((d: any) => d.toJSON())
           article.v = addons.v
         }
         article.assignIIIF()
@@ -214,8 +254,6 @@ class Service {
   }
 }
 
-module.exports = function (options) {
+export default function (options: ServiceOptions) {
   return new Service(options)
 }
-
-module.exports.Service = Service

src/services/articles/articles.service.ts

@@ -2,9 +2,9 @@ import { ServiceOptions } from '@feathersjs/feathers'
 import { createSwaggerServiceOptions } from 'feathers-swagger'
 import { ImpressoApplication } from '../../types'
 import { docs } from './articles.schema'
+import createService from './articles.class'
 
 // Initializes the `articles` service on path `/articles`
-const createService = require('./articles.class')
 const hooks = require('./articles.hooks')
 
 export default function (app: ImpressoApplication) {

src/services/search/search.extractors.js → src/services/search/search.extractors.ts

@@ -1,12 +1,20 @@
-const { keyBy, isEmpty, assignIn, clone, isUndefined, fromPairs } = require('lodash')
-const Article = require('../../models/articles.model')
-const Newspaper = require('../../models/newspapers.model')
-const Topic = require('../../models/topics.model')
-const Entity = require('../../models/entities.model')
-const Year = require('../../models/years.model')
-const { getRegionCoordinatesFromDocument } = require('../../util/solr')
-
-function getAricleMatchesAndRegions(article, documentsIndex, fragmentsIndex, highlightingIndex) {
+import { keyBy, isEmpty, assignIn, clone, isUndefined, fromPairs } from 'lodash'
+import Article from '../../models/articles.model'
+import Newspaper from '../../models/newspapers.model'
+import Topic from '../../models/topics.model'
+import Entity from '../../models/entities.model'
+import Year from '../../models/years.model'
+import { filtersToQueryAndVariables, getRegionCoordinatesFromDocument } from '../../util/solr'
+import { Service } from '../articles/articles.class'
+
+function getAricleMatchesAndRegions(
+  article: Article | undefined,
+  documentsIndex: Record<string, any>,
+  fragmentsIndex: Record<string, any>,
+  highlightingIndex: Record<string, any>
+) {
+  if (article == null) return [[], []]
+
   const { uid: id, language } = article
 
   const fragments = fragmentsIndex[id][`content_txt_${language}`]
@@ -18,10 +26,10 @@ function getAricleMatchesAndRegions(article, documentsIndex, fragmentsIndex, hig
     fragments,
   })
 
-  const regionCoords = getRegionCoordinatesFromDocument(documentsIndex[id])
+  const regionCoords: any[] = getRegionCoordinatesFromDocument(documentsIndex[id])
   const regions = Article.getRegions({
     regionCoords,
-  })
+  } as any)
 
   return [matches, regions]
 }
@@ -35,35 +43,46 @@ function getAricleMatchesAndRegions(article, documentsIndex, fragmentsIndex, hig
  *
  * @return {array} a list of `Article` items.
  */
-async function getItemsFromSolrResponse(response, articlesService, userInfo = {}) {
+export async function getItemsFromSolrResponse(
+  response: any,
+  articlesService: Service,
+  userInfo: { user?: any; authenticated?: boolean } = {}
+) {
   const { user, authenticated } = userInfo
 
   const documentsIndex = keyBy(response.response.docs, 'id')
-  const uids = response.response.docs.map(d => d.id)
+  const uids = response.response.docs.map((d: { id: string }) => d.id)
 
   if (isEmpty(uids)) return []
 
   const { fragments: fragmentsIndex, highlighting: highlightingIndex } = response
 
+  const filters = [{ type: 'uid', q: uids }]
+  const { query } = filtersToQueryAndVariables(filters)
+
   const articlesRequest = {
     user,
     authenticated,
     query: {
       limit: uids.length,
-      filters: [{ type: 'uid', q: uids }],
+      filters,
+      sq: query,
     },
   }
 
-  const articlesIndex = keyBy((await articlesService._find(articlesRequest)).data, 'uid')
+  const articlesIndex = keyBy((await articlesService.findInternal(articlesRequest)).data, 'uid')
 
-  return uids.map(uid => {
+  return uids.map((uid: string) => {
     const article = articlesIndex[uid]
     const [matches, regions] = getAricleMatchesAndRegions(article, documentsIndex, fragmentsIndex, highlightingIndex)
     return Article.assignIIIF(assignIn(clone(article), { matches, regions }))
   })
 }
 
-async function addCachedItems(bucket, provider) {
+async function addCachedItems(
+  bucket: { val: any },
+  provider: typeof Newspaper | typeof Topic | typeof Entity | typeof Year
+) {
   if (isUndefined(provider)) return bucket
   return {
     ...bucket,
@@ -85,14 +104,16 @@ const CacheProvider = {
  * @param {object} response Solr response
  * @return {object} facets object.
  */
-async function getFacetsFromSolrResponse(response) {
+export async function getFacetsFromSolrResponse(response: { facets?: Record<string, { buckets?: object[] }> }) {
   const { facets = {} } = response
 
   const facetPairs = await Promise.all(
     Object.keys(facets).map(async facetLabel => {
       if (!facets[facetLabel].buckets) return [facetLabel, facets[facetLabel]]
-      const cacheProvider = CacheProvider[facetLabel]
-      const buckets = await Promise.all(facets[facetLabel].buckets.map(async b => addCachedItems(b, cacheProvider)))
+      const cacheProvider = CacheProvider[facetLabel as keyof typeof CacheProvider]
+      const buckets = await Promise.all(
+        facets[facetLabel].buckets.map(async (b: any) => addCachedItems(b, cacheProvider))
+      )
 
       return [facetLabel, assignIn(clone(facets[facetLabel]), { buckets })]
     })
@@ -106,12 +127,6 @@ async function getFacetsFromSolrResponse(response) {
  * @param {object} response Solr response.
  * @return {number}
  */
-function getTotalFromSolrResponse(response) {
-  return response.response.numFound
-}
-
-module.exports = {
-  getItemsFromSolrResponse,
-  getFacetsFromSolrResponse,
-  getTotalFromSolrResponse,
+export function getTotalFromSolrResponse(response?: { response?: { numFound?: number } }) {
+  return response?.response?.numFound ?? 0
 }