From 312e5eddcf672894d9ebb320aec69d8fbcf08e6f Mon Sep 17 00:00:00 2001 From: aptx1231 <35984903+aptx1231@users.noreply.github.com> Date: Tue, 26 Apr 2022 18:23:30 +0800 Subject: [PATCH] fix bug in test_model and hyper (#277) * fix bug in test_model and hyper * fix pipeline * add hyper example * unify example --- hyper_example.json | 6 ++++++ hyper_example.txt | 1 + hyper_tune.py | 4 ++-- libcity/pipeline/pipeline.py | 20 +++++++++++++++----- run_hyper.py | 17 ++++++++++++----- test_model.py | 35 ++++++++++++++++++++++++----------- unit_test.py | 6 +++--- 7 files changed, 63 insertions(+), 26 deletions(-) create mode 100644 hyper_example.json create mode 100644 hyper_example.txt diff --git a/hyper_example.json b/hyper_example.json new file mode 100644 index 00000000..4da7fb28 --- /dev/null +++ b/hyper_example.json @@ -0,0 +1,6 @@ +{ + "learning_rate": { + "type": "choice", + "list": [0.01, 0.005, 0.001] + } +} \ No newline at end of file diff --git a/hyper_example.txt b/hyper_example.txt new file mode 100644 index 00000000..d3635ed1 --- /dev/null +++ b/hyper_example.txt @@ -0,0 +1 @@ +learning_rate choice [0.01, 0.005, 0.001] \ No newline at end of file diff --git a/hyper_tune.py b/hyper_tune.py index 5ca667d6..eda8b266 100644 --- a/hyper_tune.py +++ b/hyper_tune.py @@ -1,5 +1,5 @@ """ -训练并评估单一模型的脚本 +模型调参脚本 (based on the ray[tune]) """ import argparse @@ -20,7 +20,7 @@ parser.add_argument('--config_file', type=str, default=None, help='the file name of config file') parser.add_argument('--space_file', type=str, - default=None, help='the file which specifies the parameter search space') + default='hyper_example', help='the file which specifies the parameter search space') parser.add_argument('--scheduler', type=str, default='FIFO', help='the trial sheduler which will be used in ray.tune.run') parser.add_argument('--search_alg', type=str, diff --git a/libcity/pipeline/pipeline.py b/libcity/pipeline/pipeline.py index 24c19c1a..78390b2d 100644 --- a/libcity/pipeline/pipeline.py +++ b/libcity/pipeline/pipeline.py @@ -136,8 +136,15 @@ def hyper_parameter(task=None, model_name=None, dataset_name=None, config_file=N # load config experiment_config = ConfigParser(task, model_name, dataset_name, config_file=config_file, other_args=other_args) + # exp_id + exp_id = experiment_config.get('exp_id', None) + if exp_id is None: + exp_id = int(random.SystemRandom().random() * 100000) + experiment_config['exp_id'] = exp_id # logger logger = get_logger(experiment_config) + logger.info('Begin ray-tune, task={}, model_name={}, dataset_name={}, exp_id={}'. + format(str(task), str(model_name), str(dataset_name), str(exp_id))) logger.info(experiment_config.config) # check space_file if space_file is None: @@ -167,8 +174,11 @@ def train(config, checkpoint_dir=None, experiment_config=None, experiment_config[key] = config[key] experiment_config['hyper_tune'] = True logger = get_logger(experiment_config) - logger.info('Begin pipeline, task={}, model_name={}, dataset_name={}' - .format(str(task), str(model_name), str(dataset_name))) + # exp_id + exp_id = int(random.SystemRandom().random() * 100000) + experiment_config['exp_id'] = exp_id + logger.info('Begin pipeline, task={}, model_name={}, dataset_name={}, exp_id={}'. + format(str(task), str(model_name), str(dataset_name), str(exp_id))) logger.info('running parameters: ' + str(config)) # load model model = get_model(experiment_config, data_feature) @@ -215,9 +225,9 @@ def train(config, checkpoint_dir=None, experiment_config=None, # save best best_path = os.path.join(best_trial.checkpoint.value, "checkpoint") model_state, optimizer_state = torch.load(best_path) - model_cache_file = './libcity/cache/model_cache/{}_{}.m'.format( - model_name, dataset_name) - ensure_dir('./libcity/cache/model_cache') + model_cache_file = './libcity/cache/{}/model_cache/{}_{}.m'.format( + exp_id, model_name, dataset_name) + ensure_dir('./libcity/cache/{}/model_cache'.format(exp_id)) torch.save((model_state, optimizer_state), model_cache_file) diff --git a/run_hyper.py b/run_hyper.py index aa40e579..5e3869c4 100644 --- a/run_hyper.py +++ b/run_hyper.py @@ -1,9 +1,9 @@ """ -单一模型调参脚本 +模型调参脚本 (based on the hyperopt) """ import argparse - +import random from libcity.pipeline import objective_function from libcity.executor import HyperTuning from libcity.utils import str2bool, get_logger, set_random_seed, add_general_args @@ -26,7 +26,7 @@ help='whether re-train model if the model is \ trained before') parser.add_argument('--params_file', type=str, - default=None, help='the file which specify the \ + default='hyper_example.txt', help='the file which specify the \ hyper-parameters and ranges to be adjusted') parser.add_argument('--hyper_algo', type=str, default='grid_search', help='hyper-parameters search algorithm') @@ -43,11 +43,18 @@ other_args = {key: val for key, val in dict_args.items() if key not in [ 'task', 'model', 'dataset', 'config_file', 'saved_model', 'train', 'params_file', 'hyper_algo'] and val is not None} - - logger = get_logger({'model': args.model, 'dataset': args.dataset}) + # exp_id + exp_id = dict_args.get('exp_id', None) + if exp_id is None: + # Make a new experiment ID + exp_id = int(random.SystemRandom().random() * 100000) + other_args['exp_id'] = exp_id + # logger + logger = get_logger({'model': args.model, 'dataset': args.dataset, 'exp_id': exp_id}) # seed seed = dict_args.get('seed', 0) set_random_seed(seed) + other_args['seed'] = seed hp = HyperTuning(objective_function, params_file=args.params_file, algo=args.hyper_algo, max_evals=args.max_evals, task=args.task, model_name=args.model, dataset_name=args.dataset, config_file=args.config_file, diff --git a/test_model.py b/test_model.py index 83a7b243..2ed17b07 100644 --- a/test_model.py +++ b/test_model.py @@ -1,14 +1,26 @@ from libcity.config import ConfigParser from libcity.data import get_dataset -from libcity.utils import get_model, get_executor +from libcity.utils import get_model, get_executor, get_logger, set_random_seed +import random + +""" +取一个batch的数据进行初步测试 +Take the data of a batch for preliminary testing +""" # 加载配置文件 -config = ConfigParser(task='traj_loc_pred', model='TemplateTLP', - dataset='foursquare_tky', config_file=None, - other_args={'batch_size': 2}) -# 如果是交通流量\速度预测任务，请使用下面的加载配置文件语句 -# config = ConfigParser(task='traffic_state_pred', model='TemplateTSP', -# dataset='METR_LA', config_file=None, other_args={'batch_size': 2}) +config = ConfigParser(task='traffic_state_pred', model='RNN', + dataset='METR_LA', other_args={'batch_size': 2}) +exp_id = config.get('exp_id', None) +if exp_id is None: + exp_id = int(random.SystemRandom().random() * 100000) + config['exp_id'] = exp_id +# logger +logger = get_logger(config) +logger.info(config.config) +# seed +seed = config.get('seed', 0) +set_random_seed(seed) # 加载数据模块 dataset = get_dataset(config) # 数据预处理，划分数据集 @@ -18,10 +30,11 @@ batch = train_data.__iter__().__next__() # 加载模型 model = get_model(config, data_feature) -self = model.to(config['device']) +model = model.to(config['device']) +# 加载执行器 +executor = get_executor(config, model, data_feature) # 模型预测 batch.to_tensor(config['device']) res = model.predict(batch) -# 请自行确认 res 的 shape 是否符合赛道的约束 -# 如果要加载执行器的话 -executor = get_executor(config, model) +logger.info('Result shape is {}'.format(res.shape)) +logger.info('Success test the model!') diff --git a/unit_test.py b/unit_test.py index b67cabf9..fe21a825 100644 --- a/unit_test.py +++ b/unit_test.py @@ -5,10 +5,10 @@ ############################################# # The parameter to control the unit testing # -tested_trajectory_model = 'TemplateTLP' -tested_trajectory_dataset = 'foursquare_tky' +tested_trajectory_model = 'RNN' +tested_trajectory_dataset = 'foursquare_nyc' tested_trajectory_encoder = 'StandardTrajectoryEncoder' -tested_traffic_state_model = 'DCRNN' +tested_traffic_state_model = 'RNN' tested_traffic_state_dataset = 'METR_LA' #############################################