diff --git a/delta/app/src/main/resources/app.conf b/delta/app/src/main/resources/app.conf index 4f788d00cc..ee2534a173 100644 --- a/delta/app/src/main/resources/app.conf +++ b/delta/app/src/main/resources/app.conf @@ -39,6 +39,7 @@ app { # this provides 2-3x performance improvement rewrite-batch-inserts = true + # Cache for the underlying Postgres partitions cache { # The max number of tokens in the partition cache max-size = 1000 @@ -50,29 +51,51 @@ app { slow-query-threshold = 2 seconds } + # Query configuration for view indexing operations elem-query { + # Type of behaviour to adopt when all elements have been indexed + # - delay: wait for a fixed delay before querying the primary store again + # - passivation: based on the `project-last-update` feature. When selected: + # - for active projects, waits for the fixed delay before retrying + # - for inactive projects, waits for `project-last-update` to notify updates + # of the project to resume, decreasing significantly the number of queries to + # the primary store type = "delay" + # The maximum number of elements to retrieve in a single query batch-size = 30 + # Fixed delay to wait before attempting to fetch new elements when the current ones have + # already been processed delay = 2s } + # Allows to compute incrementally the last update instant and ordering value for each project + # Split in two parts: + # - a write one running on a single node to compute and save in the database the new values + # - a read one running on all nodes, which retrieves those values and notify back indexing routines + # a project is active again project-last-update { + # Configuration for the write part batch { # the maximum batching size, corresponding to the maximum number of elements being aggregated # at the same time before pushing the update. max-elements = 100 - # the maximum batching duration. + # the maximum amount of time before pushing the update max-interval = 1 second } + # Configuration for the read part query { + # Maximum number of rows to read back batch-size = 30 + # How much time before attempting to get new updates refresh-strategy = 1 second } + # Only projects with a last update instant contained in this interval inactive-interval = 10 minutes } # Database export configuration export { + # how many rows to get per call batch-size = 30 # Limit number of events per files (this default value should give ~1GB files) limit-per-file = 32000 @@ -331,7 +354,9 @@ app { # projection configuration projections { cluster { + # Number of nodes in the cluster size = 1 + # Index of the current Delta instance in the cluster node-index = 0 } batch { diff --git a/delta/plugins/blazegraph/src/main/resources/blazegraph.conf b/delta/plugins/blazegraph/src/main/resources/blazegraph.conf index aa9fc9e4dd..d423a7ca3b 100644 --- a/delta/plugins/blazegraph/src/main/resources/blazegraph.conf +++ b/delta/plugins/blazegraph/src/main/resources/blazegraph.conf @@ -16,6 +16,7 @@ plugins.blazegraph { # Blazegraph query timeout query-timeout = "1 minute" + slow-queries { # How long a query takes before it is considered slow (and hence, logged) slow-query-threshold = 30 seconds @@ -34,11 +35,9 @@ plugins.blazegraph { max-view-refs = 20 #the maximum duration allowed so that synchronous indexing can complete sync-indexing-timeout = 10 seconds - # In order to disable this feature, set an infinite time ('Inf') - # idle-timeout = Inf batch { # the maximum batching size, corresponding to the maximum number of elements - # uploaded on a bulk request. + # pushed in Blazegraph in a bulk update request. max-elements = 10 # the maximum batching duration. max-interval = 3 seconds diff --git a/delta/plugins/composite-views/src/main/resources/composite-views.conf b/delta/plugins/composite-views/src/main/resources/composite-views.conf index 3f285ec975..9c058c654f 100644 --- a/delta/plugins/composite-views/src/main/resources/composite-views.conf +++ b/delta/plugins/composite-views/src/main/resources/composite-views.conf @@ -43,7 +43,11 @@ plugins.composite-views { # the interval at which a view will look for requested restarts restart-check-interval = 3 seconds + # Batch configuration to push data to Blazegraph for both sources and projections + # Defaults to the one defined in the Blazegraph plugin blazegraph-batch = ${plugins.blazegraph.batch} + + # Batch configuration to push data to Elasticsearch for the related projections elasticsearch-batch = ${plugins.elasticsearch.batch} # set to false to disable composite view indexing diff --git a/delta/plugins/elasticsearch/src/main/resources/elasticsearch.conf b/delta/plugins/elasticsearch/src/main/resources/elasticsearch.conf index b467d24983..56cb4d1079 100644 --- a/delta/plugins/elasticsearch/src/main/resources/elasticsearch.conf +++ b/delta/plugins/elasticsearch/src/main/resources/elasticsearch.conf @@ -18,8 +18,6 @@ plugins.elasticsearch { prefix = ${app.defaults.indexing.prefix} # configuration of the maximum number of view references allowed on an aggregated view max-view-refs = 20 - # In order to disable this feature, set an infinite time ('Inf') - # idle-timeout = Inf #the maximum duration allowed so that synchronous indexing can complete sync-indexing-timeout = 10 seconds #the value for `refresh` Elasticsearch parameter for synchronous indexing