From 117f67d6e1eac9e780d3bdad7e02a77d46c2b97c Mon Sep 17 00:00:00 2001 From: Linlang Date: Thu, 23 May 2024 06:57:59 +0800 Subject: [PATCH] fix get data error --- examples/orderbook_data/README.md | 5 +++-- qlib/tests/data.py | 14 +++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/examples/orderbook_data/README.md b/examples/orderbook_data/README.md index 890e11f41e..53fd523d7f 100644 --- a/examples/orderbook_data/README.md +++ b/examples/orderbook_data/README.md @@ -16,7 +16,7 @@ Current version of script with default value tries to connect localhost **via de Run following command to install necessary libraries ``` -pip install pytest coverage +pip install pytest coverage gdown pip install arctic # NOTE: pip may fail to resolve the right package dependency !!! Please make sure the dependency are satisfied. ``` @@ -27,7 +27,8 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency 2. Please follow following steps to download example data ```bash cd examples/orderbook_data/ -python ../../scripts/get_data.py download_data --target_dir . --file_name highfreq_orderbook_example_data.zip +gdown https://drive.google.com/uc?id=15nZF7tFT_eKVZAcMFL1qPS4jGyJflH7e # Proxies may be necessary here. +python ../../scripts/get_data.py _unzip --file_path highfreq_orderbook_example_data.zip --target_dir . ``` 3. Please import the example data to your mongo db diff --git a/qlib/tests/data.py b/qlib/tests/data.py index f6bd780905..0d53b2dc62 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -12,15 +12,11 @@ from tqdm import tqdm from pathlib import Path from loguru import logger -from cryptography.fernet import Fernet from qlib.utils import exists_qlib_data class GetData: - REMOTE_URL = "https://qlibpublic.blob.core.windows.net/data/default/stock_data" - # "?" is not included in the token. - TOKEN = b"gAAAAABkmDhojHc0VSCDdNK1MqmRzNLeDFXe5hy8obHpa6SDQh4de6nW5gtzuD-fa6O_WZb0yyqYOL7ndOfJX_751W3xN5YB4-n-P22jK-t6ucoZqhT70KPD0Lf0_P328QPJVZ1gDnjIdjhi2YLOcP4BFTHLNYO0mvzszR8TKm9iT5AKRvuysWnpi8bbYwGU9zAcJK3x9EPL43hOGtxliFHcPNGMBoJW4g_ercdhi0-Qgv5_JLsV-29_MV-_AhuaYvJuN2dEywBy" - KEY = "EYcA8cgorA8X9OhyMwVfuFxn_1W3jGk6jCbs3L2oPoA=" + REMOTE_URL = "https://github.com/SunsetWolf/qlib_dataset/releases/download" def __init__(self, delete_zip_file=False): """ @@ -33,9 +29,7 @@ def __init__(self, delete_zip_file=False): self.delete_zip_file = delete_zip_file def merge_remote_url(self, file_name: str): - fernet = Fernet(self.KEY) - token = fernet.decrypt(self.TOKEN).decode() - return f"{self.REMOTE_URL}/{file_name}?{token}" + return f"{self.REMOTE_URL}/{file_name}" def download_data(self, file_name: str, target_dir: [Path, str], delete_old: bool = True): """ @@ -99,7 +93,9 @@ def check_dataset(self, file_name: str): return status @staticmethod - def _unzip(file_path: Path, target_dir: Path, delete_old: bool = True): + def _unzip(file_path: [Path, str], target_dir: [Path, str], delete_old: bool = True): + file_path = Path(file_path) + target_dir = Path(target_dir) if delete_old: logger.warning( f"will delete the old qlib data directory(features, instruments, calendars, features_cache, dataset_cache): {target_dir}"