From 77fc657172300a98148a2660c3f645f6d447ec84 Mon Sep 17 00:00:00 2001 From: Abit Date: Mon, 18 Oct 2021 23:02:03 +0200 Subject: [PATCH 1/3] Fixes get_markets and get_daily_volume Get volume data from more reasonable fields --- services/bitshares_elasticsearch_client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/bitshares_elasticsearch_client.py b/services/bitshares_elasticsearch_client.py index a296ffc..135dc8c 100644 --- a/services/bitshares_elasticsearch_client.py +++ b/services/bitshares_elasticsearch_client.py @@ -45,21 +45,21 @@ def get_markets(self, from_date, to_date, base=None, quote=None): "composite" : { "size": 10000, # TODO use a generator function instead of a big size, see https://github.com/elastic/elasticsearch-dsl-py/blob/master/examples/composite_agg.py#L21 "sources" : [ - { "base": { "terms" : { "field": "operation_history.op_object.fill_price.base.asset_id.keyword" } } }, - { "quote": { "terms" : { "field": "operation_history.op_object.fill_price.quote.asset_id.keyword" } } } + { "base": { "terms" : { "field": "operation_history.op_object.pays.asset_id.keyword" } } }, + { "quote": { "terms" : { "field": "operation_history.op_object.receives.asset_id.keyword" } } } ] }, "aggs": { - "volume": { "sum" : { "field" : "operation_history.op_object.fill_price.quote.amount" } } + "volume": { "sum" : { "field" : "operation_history.op_object.receives.amount" } } } } } } if base: - query['query']['bool']['filter'].append({ "term": { "operation_history.op_object.fill_price.base.asset_id.keyword": base } }) + query['query']['bool']['filter'].append({ "term": { "operation_history.op_object.pays.asset_id.keyword": base } }) if quote: - query['query']['bool']['filter'].append({ "term": { "operation_history.op_object.fill_price.quote.asset_id.keyword": quote } }) + query['query']['bool']['filter'].append({ "term": { "operation_history.op_object.receives.asset_id.keyword": quote } }) client = connections.get_connection('operations') response = client.search(index="bitshares-*", body=query) @@ -124,11 +124,11 @@ def get_daily_volume(self, from_date, to_date): s = s.query('bool', filter = [ Q('term', operation_type=4), Q('range', block_data__block_time={'gte': from_date, 'lte': to_date}), - Q('term', operation_history__op_object__fill_price__quote__asset_id__keyword=config.CORE_ASSET_ID) + Q('term', operation_history__op_object__receives__asset_id__keyword=config.CORE_ASSET_ID) ]) a = A('date_histogram', field='block_data.block_time', interval='1d', format='yyyy-MM-dd') \ - .metric('volume', 'sum', field='operation_history.op_object.fill_price.quote.amount') + .metric('volume', 'sum', field='operation_history.op_object.receives.amount') s.aggs.bucket('volume_over_time', a) response = s.execute() From 6933229b52d7161dec779f682d364c8e285833e4 Mon Sep 17 00:00:00 2001 From: Abit Date: Mon, 18 Oct 2021 23:16:06 +0200 Subject: [PATCH 2/3] Add notes --- services/bitshares_elasticsearch_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/bitshares_elasticsearch_client.py b/services/bitshares_elasticsearch_client.py index 135dc8c..1685636 100644 --- a/services/bitshares_elasticsearch_client.py +++ b/services/bitshares_elasticsearch_client.py @@ -28,7 +28,7 @@ def get_markets(self, from_date, to_date, base=None, quote=None): "query": { "bool": { "filter": [ - { "term": { "operation_type": 4 } }, + { "term": { "operation_type": 4 } }, # NOTE: may logically return duplicate data since not filtering by `is_maker == true` { "range": { "block_data.block_time": { @@ -51,6 +51,8 @@ def get_markets(self, from_date, to_date, base=None, quote=None): }, "aggs": { "volume": { "sum" : { "field" : "operation_history.op_object.receives.amount" } } + # NOTE: perhaps better return both `pays.amount` and `receives.amount` (in different fields but not add them together), + # because it does not make much sense to return only `receives.amount` when filtering by `pays.asset_id`. } } } From 98fff1fb8491d91127fef69acc3f0b672de6120c Mon Sep 17 00:00:00 2001 From: Abit Date: Tue, 19 Oct 2021 01:50:46 +0200 Subject: [PATCH 3/3] Update a comment --- services/bitshares_elasticsearch_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/bitshares_elasticsearch_client.py b/services/bitshares_elasticsearch_client.py index 1685636..d2f6924 100644 --- a/services/bitshares_elasticsearch_client.py +++ b/services/bitshares_elasticsearch_client.py @@ -28,7 +28,7 @@ def get_markets(self, from_date, to_date, base=None, quote=None): "query": { "bool": { "filter": [ - { "term": { "operation_type": 4 } }, # NOTE: may logically return duplicate data since not filtering by `is_maker == true` + { "term": { "operation_type": 4 } }, # NOTE: two fill_order_operation entries for each match, one for each side { "range": { "block_data.block_time": {