Skip to content

Commit

Permalink
Deconfigures indexing for clarity. (#97)
Browse files Browse the repository at this point in the history 
Deconfigures indexing for clarity.
  • Loading branch information
@mjgiarlo
mjgiarlo authored Oct 30, 2019
2 parents eb3ca03 + a0c4db7 commit a844f8f
Show file tree
Hide file tree
Showing 7 changed files with 128 additions and 248 deletions.
8 changes: 0 additions & 8 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,6 @@ $ curl -i -X POST -H 'Content-Type: application/ld+json' -H 'Link: <http://www.w

See [Sinopia Server notes](https://github.com/LD4P/sinopia_server/wiki/Draft-Notes-for-Sinopia-Server-API-Spec) for more Trellis `curl` incantations.

### Inspect indexing behavior

While testing the pipeline, it may be useful for you to run test searches against ElasticSearch to inspect the resulting index entries that the pipeline created. To do this, a `npm run search` script has been included:

```shell
$ npm run search A Tale of Two Cities
```

## Development

For development purposes, you may wish to spin up all the components other than the pipeline if you'll be iterating:
Expand Down
18 changes: 6 additions & 12 deletions __tests__/Indexer.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,7 @@ describe('Indexer', () => {
'There is nothing either good or bad, but thinking makes it so.'
],
subtitle: ['A Tragic Tale about a Prince of Denmark'],
'subtitle-suggest': ['a', 'tragic', 'tale', 'about', 'a', 'prince', 'of', 'denmark'],
title: ['Hamlet'],
'title-suggest': ['hamlet'],
type: ['http://id.loc.gov/ontologies/bibframe/AbbreviatedTitle'],
uri: 'http://foo.bar/12345',
created: '2019-10-18T16:08:43.300Z',
Expand Down Expand Up @@ -212,7 +210,6 @@ describe('Indexer', () => {
'n'
],
title: ['What factors influence the quality of hazard mitigation plans in Washington State?'],
'title-suggest': ['what', 'factors', 'influence', 'the', 'quality', 'of', 'hazard', 'mitigation', 'plans', 'in', 'washington', 'state?'],
type: ['http://rdaregistry.info/Elements/c/C10006'],
uri: 'http://foo.bar/12345',
created: '2019-10-23T15:40:51.049Z',
Expand Down Expand Up @@ -357,9 +354,7 @@ describe('Indexer', () => {
'There is nothing either good or bad, but thinking makes it so.'
],
subtitle: ['A Tragic Tale about a Prince of Denmark'],
'subtitle-suggest': ['a', 'tragic', 'tale', 'about', 'a', 'prince', 'of', 'denmark'],
title: ['Hamlet'],
'title-suggest': ['hamlet'],
type:[
[ 'http://id.loc.gov/ontologies/bibframe/AbbreviatedTitle',
'http://id.loc.gov/ontologies/bibframe/WorkTitle'
Expand Down Expand Up @@ -519,9 +514,8 @@ describe('Indexer', () => {

it('does not log an error', async () => {
await indexer.setupIndices()
const indexCount = Object.keys(config.get('indexMappings')).length
expect(createSpy).toHaveBeenCalledTimes(indexCount)
expect(mappingSpy).toHaveBeenCalledTimes(indexCount)
expect(createSpy).toHaveBeenCalledTimes(2)
expect(mappingSpy).toHaveBeenCalledTimes(2)
expect(logSpy).not.toHaveBeenCalled()
})
})
Expand Down Expand Up @@ -574,16 +568,16 @@ describe('Indexer', () => {

describe('indexFrom()', () => {
it('returns the resource index name by default', () => {
expect(indexer.indexFrom([])[0]).toBe('sinopia_resources')
expect(indexer.indexFrom([])).toBe('sinopia_resources')
})
it('returns the resource index name for a resource', () => {
expect(indexer.indexFrom(resourceObjectTypes)[0]).toBe('sinopia_resources')
expect(indexer.indexFrom(resourceObjectTypes)).toBe('sinopia_resources')
})
it('returns the non RDF index name when types includes LDP-NRS', () => {
expect(indexer.indexFrom(['http://www.w3.org/ns/ldp#NonRDFSource'])[0]).toBe('sinopia_templates')
expect(indexer.indexFrom(['http://www.w3.org/ns/ldp#NonRDFSource'])).toBe('sinopia_templates')
})
it('returns undefined for a container', () => {
expect(indexer.indexFrom(containerObjectTypes)[0]).toBe(undefined)
expect(indexer.indexFrom(containerObjectTypes)).toBe(undefined)
})
})
})
61 changes: 0 additions & 61 deletions config/default.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,67 +15,6 @@ module.exports = {
defaultMimeType: process.env.DEFAULT_MIME_TYPE || 'application/ld+json',
indexType: process.env.INDEX_TYPE || 'sinopia',
indexUrl: process.env.INDEX_URL || 'http://localhost:9200',
// Note that InputLookupSinopia expects uri and label fields.
indexMappings: process.env.INDEX_FIELD_MAPPINGS
? JSON.parse(process.env.INDEX_FIELD_MAPPINGS)
: {
sinopia_resources: {
store_document: false,
fields: {
title: {
type: 'text',
path: '$..[mainTitle,P10223,P20315,P40085,P30156]', //BIBFRAME and RDA
autosuggest: true
},
subtitle: {
type: 'text',
path: '$..subtitle',
autosuggest: true
},
type: {
type: 'keyword',
store: true
},
uri: {
type: 'keyword',
id: true,
store: true,
index: true
},
label: {
type: 'keyword',
// If not title, then URI.
fields: [['title', 'subtitle'], ['uri']],
joinby: ': ',
store: true,
index: false
},
created: {
type: 'date',
asTypes: ['Create'],
store: true,
index: true
},
modified: {
type: 'date',
asTypes: ['Create', 'Update'],
store: true,
index: true
},
text: {
type: 'text',
path: '$..*',
store: false,
index: true
}
}
},
// Not yet mapped
sinopia_templates: {
store_document: true,
fields: {}
}
},
nonRdfTypeURI: process.env.NON_RDF_TYPE_URI || 'http://www.w3.org/ns/ldp#NonRDFSource',
nonRdfMimeType: process.env.NON_RDF_MIME_TYPE || 'application/json',
debug: process.env.DEBUG !== undefined ? process.env.DEBUG : true
Expand Down
60 changes: 0 additions & 60 deletions search.js
View file

This file was deleted.

57 changes: 15 additions & 42 deletions src/Indexer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ export default class Indexer {
this.logger = new Logger()
this.knownIndexResults = ['created', 'updated']
this.knownDeleteResults = ['deleted']
this.indexers = {
sinopia_templates: SinopiaTemplateIndexer,
sinopia_resources: ResourceIndexer
}
}

/**
Expand All @@ -25,21 +29,20 @@ export default class Indexer {
* @returns {Promise} resolves to true if successful; null if not
*/
async index(json, uri, types) {
const [index, store_document, fields] = this.indexFrom(types)
const index = this.indexFrom(types)
this.logger.debug(`${uri} (${types}) has index ${index}`)
if (index === undefined) {

const indexer = this.indexers[index]
if (indexer === undefined) {
this.logger.debug(`skipping indexing ${uri} (${types})`)
return true
}
const indexer = index === 'sinopia_templates' ? SinopiaTemplateIndexer : ResourceIndexer

const body = new indexer(json, uri, store_document, fields).index()

return this.client.index({
index: index,
type: config.get('indexType'),
id: this.identifierFrom(uri),
body: body
body: new indexer(json, uri).index()
}).then(indexResponse => {
if (!this.knownIndexResults.includes(indexResponse.result))
throw { message: JSON.stringify(indexResponse) }
Expand All @@ -58,7 +61,7 @@ export default class Indexer {
* @param {Promise} resolves to types - one or more LDP type URIs
*/
async delete(uri, types) {
const [index] = this.indexFrom(types)
const index = this.indexFrom(types)
if (index === undefined) {
this.logger.debug(`skipping deleting ${uri} (${types})`)
return true
Expand Down Expand Up @@ -93,12 +96,10 @@ export default class Indexer {

/**
* Create indices, if needed, and add field mappings
* @returns {null}
*/
async setupIndices() {
const indexMappings = config.get('indexMappings')
try {
for (const index of Object.keys(indexMappings)) {
for (const index of Object.keys(this.indexers)) {
const indexExists = await this.client.indices.exists({ index: index })

if (!indexExists) {
Expand All @@ -111,7 +112,7 @@ export default class Indexer {
await this.client.indices.putMapping({
index: index,
type: config.get('indexType'),
body: this.buildMappingsFromConfig(indexMappings[index].fields)
body: this.indexers[index].indexMapping
})
}
} catch(error) {
Expand All @@ -120,32 +121,6 @@ export default class Indexer {
return null
}

/**
* Build field mappings from configuration
* @param {Object} fields - Field configuration
* @returns {Object}
*/
buildMappingsFromConfig(fields) {
const mappingObject = { properties: {} }

for (const [fieldName, fieldProperties] of Object.entries(fields)) {
mappingObject.properties[fieldName] = {
type: fieldProperties.type,
store: fieldProperties.store == true,
index: fieldProperties.index == false ? false : true
}

if (fieldProperties.autosuggest) {
mappingObject.properties[`${fieldName}-suggest`] = {
type: 'completion'
}
}
}

return mappingObject
}


/**
* Remove and recreate all known indices
* @returns {Promise} resolves to null upon completion (errors, if any, are logged)
Expand Down Expand Up @@ -175,12 +150,10 @@ export default class Indexer {
/**
* Returns index information given a list of LDP types.
* @param {Array} types - LDP type URIs of object
* @returns {[string, store_document, fields]} name of index | undefined if should not index, whether to store document, field configuration
* @returns {string]} name of index or undefined
*/
indexFrom(types) {
if (types.includes('http://www.w3.org/ns/ldp#BasicContainer')) return [undefined, undefined, undefined]
const indexMappings = config.get('indexMappings')
const index = types.includes(config.get('nonRdfTypeURI')) ? 'sinopia_templates' : 'sinopia_resources'
return [index, indexMappings[index].store_document, indexMappings[index].fields]
if (types.includes('http://www.w3.org/ns/ldp#BasicContainer')) return undefined
return types.includes(config.get('nonRdfTypeURI')) ? 'sinopia_templates' : 'sinopia_resources'
}
}
Loading

0 comments on commit a844f8f

Please sign in to comment.