From 580aad0349791de7a69858c70e5acd8df27884de Mon Sep 17 00:00:00 2001 From: jungsooyun Date: Wed, 12 Jan 2022 02:05:34 +0900 Subject: [PATCH] define get_block_range_for_date_hour for hourly airflow batch --- ethereumetl/cli/__init__.py | 2 + .../cli/get_block_range_for_date_hour.py | 58 +++++++++++++++++++ ethereumetl/service/eth_service.py | 7 ++- setup.py | 2 +- 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 ethereumetl/cli/get_block_range_for_date_hour.py diff --git a/ethereumetl/cli/__init__.py b/ethereumetl/cli/__init__.py index f88dd6b7f..bc8769608 100644 --- a/ethereumetl/cli/__init__.py +++ b/ethereumetl/cli/__init__.py @@ -42,6 +42,7 @@ from ethereumetl.cli.extract_tokens import extract_tokens from ethereumetl.cli.filter_items import filter_items from ethereumetl.cli.get_block_range_for_date import get_block_range_for_date +from ethereumetl.cli.get_block_range_for_date_hour import get_block_range_for_date_hour from ethereumetl.cli.get_block_range_for_timestamps import get_block_range_for_timestamps from ethereumetl.cli.get_keccak_hash import get_keccak_hash from ethereumetl.cli.stream import stream @@ -74,6 +75,7 @@ def cli(ctx): # utils cli.add_command(get_block_range_for_date, "get_block_range_for_date") +cli.add_command(get_block_range_for_date_hour, "get_block_range_for_date_hour") cli.add_command(get_block_range_for_timestamps, "get_block_range_for_timestamps") cli.add_command(get_keccak_hash, "get_keccak_hash") cli.add_command(extract_csv_column, "extract_csv_column") diff --git a/ethereumetl/cli/get_block_range_for_date_hour.py b/ethereumetl/cli/get_block_range_for_date_hour.py new file mode 100644 index 000000000..16e1ecd9e --- /dev/null +++ b/ethereumetl/cli/get_block_range_for_date_hour.py @@ -0,0 +1,58 @@ +# MIT License +# +# Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import click + +from datetime import datetime +from ethereumetl.web3_utils import build_web3 + +from blockchainetl.file_utils import smart_open +from blockchainetl.logging_utils import logging_basic_config +from ethereumetl.service.eth_service import EthService +from ethereumetl.providers.auto import get_provider_from_uri +from ethereumetl.utils import check_classic_provider_uri + +logging_basic_config() + + +@click.command(context_settings=dict(help_option_names=['-h', '--help'])) +@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, + help='The URI of the web3 provider e.g. ' + 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') +@click.option('-d', '--date', required=True, type=lambda d: datetime.strptime(d, '%Y-%m-%d'), + help='The date e.g. 2018-01-01.') +@click.option('-h', '--hour', required=True, type=int, + help='The hour e.g. 9. 23.') +@click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.') +@click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.') +def get_block_range_for_date_hour(provider_uri, date, hour, output, chain='ethereum'): + """Outputs start and end blocks for given date and hour.""" + provider_uri = check_classic_provider_uri(chain, provider_uri) + provider = get_provider_from_uri(provider_uri) + web3 = build_web3(provider) + eth_service = EthService(web3) + + start_block, end_block = eth_service.get_block_range_for_date_hour(date, hour) + + with smart_open(output, 'w') as output_file: + output_file.write('{},{}\n'.format(start_block, end_block)) diff --git a/ethereumetl/service/eth_service.py b/ethereumetl/service/eth_service.py index cd12cea17..9ba2e7306 100644 --- a/ethereumetl/service/eth_service.py +++ b/ethereumetl/service/eth_service.py @@ -21,7 +21,7 @@ # SOFTWARE. -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from ethereumetl.service.graph_operations import GraphOperations, OutOfBoundsError, Point @@ -36,6 +36,11 @@ def get_block_range_for_date(self, date): end_datetime = datetime.combine(date, datetime.max.time().replace(tzinfo=timezone.utc)) return self.get_block_range_for_timestamps(start_datetime.timestamp(), end_datetime.timestamp()) + def get_block_range_for_date_hour(self, date, hour: int): + start_hour = datetime.combine(date, datetime.min.time().replace(tzinfo=timezone.utc)) + timedelta(hours=hour) + end_hour = start_hour + timedelta(hours=1) - datetime.resolution + return self.get_block_range_for_timestamps(start_hour.timestamp(), end_hour.timestamp()) + def get_block_range_for_timestamps(self, start_timestamp, end_timestamp): start_timestamp = int(start_timestamp) end_timestamp = int(end_timestamp) diff --git a/setup.py b/setup.py index 040bc5c1e..61e804865 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def read(fname): setup( name='ethereum-etl', - version='1.10.0', + version='jerry', author='Evgeny Medvedev', author_email='evge.medvedev@gmail.com', description='Tools for exporting Ethereum blockchain data to CSV or JSON',