diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 48edeb9a5..dd26ebf99 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -121,6 +121,9 @@ class BaseTask: exclude_components (Optional[Dict]): If None, all possible components are used. Otherwise specifies set of components not to use. Incompatible with include components + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline """ def __init__( @@ -697,6 +700,7 @@ def _search( precision: int = 32, disable_file_output: List = [], load_models: bool = True, + portfolio_selection: Optional[str] = None ) -> 'BaseTask': """ Search for the best pipeline configuration for the given dataset. @@ -767,7 +771,15 @@ def _search( disable_file_output (Union[bool, List]): load_models (bool), (default=True): Whether to load the models after fitting AutoPyTorch. - + portfolio_selection (str), (default=None): + This argument controls the initial configurations that + AutoPyTorch uses to warm start SMAC for hyperparameter + optimization. By default, no warm-starting happens. + The user can provide a path to a json file containing + configurations, similar to (...herepathtogreedy...). + Additionally, the keyword 'greedy' is supported, + which would use the default portfolio from + `AutoPyTorch Tabular ` Returns: self @@ -955,7 +967,8 @@ def _search( # We do not increase the num_run here, this is something # smac does internally start_num_run=self._backend.get_next_num_run(peek=True), - search_space_updates=self.search_space_updates + search_space_updates=self.search_space_updates, + portfolio_selection=portfolio_selection, ) try: run_history, self.trajectory, budget_type = \ diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 9662afd67..9da96ef94 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -56,6 +56,9 @@ class TabularClassificationTask(BaseTask): If None, all possible components are used. Otherwise specifies set of components not to use. Incompatible with include components + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline """ def __init__( self, @@ -119,6 +122,7 @@ def search( precision: int = 32, disable_file_output: List = [], load_models: bool = True, + portfolio_selection: Optional[str] = None, ) -> 'BaseTask': """ Search for the best pipeline configuration for the given dataset. @@ -131,8 +135,8 @@ def search( A pair of features (X_train) and targets (y_train) used to fit a pipeline. Additionally, a holdout of this pairs (X_test, y_test) can be provided to track the generalization performance of each stage. - optimize_metric (str): name of the metric that is used to - evaluate a pipeline. + optimize_metric (str): + name of the metric that is used to evaluate a pipeline. budget_type (Optional[str]): Type of budget to be used when fitting the pipeline. Either 'epochs' or 'runtime'. If not provided, uses @@ -140,12 +144,12 @@ def search( budget (Optional[float]): Budget to fit a single run of the pipeline. If not provided, uses the default in the pipeline config - total_walltime_limit (int), (default=100): Time limit - in seconds for the search of appropriate models. + total_walltime_limit (int), (default=100): + Time limit in seconds for the search of appropriate models. By increasing this value, autopytorch has a higher chance of finding better models. - func_eval_time_limit_secs (int), (default=None): Time limit - for a single call to the machine learning model. + func_eval_time_limit_secs (int), (default=None): + Time limit for a single call to the machine learning model. Model fitting will be terminated if the machine learning algorithm runs over the time limit. Set this value high enough so that typical machine @@ -162,32 +166,40 @@ def search( feature by turning this flag to False. All machine learning algorithms that are fitted during search() are considered for ensemble building. - memory_limit (Optional[int]), (default=4096): Memory - limit in MB for the machine learning algorithm. autopytorch + memory_limit (Optional[int]), (default=4096): + Memory limit in MB for the machine learning algorithm. autopytorch will stop fitting the machine learning algorithm if it tries to allocate more than memory_limit MB. If None is provided, no memory limit is set. In case of multi-processing, memory_limit will be per job. This memory limit also applies to the ensemble creation process. - smac_scenario_args (Optional[Dict]): Additional arguments inserted - into the scenario of SMAC. See the + smac_scenario_args (Optional[Dict]): + Additional arguments inserted into the scenario of SMAC. See the [SMAC documentation] (https://automl.github.io/SMAC3/master/options.html?highlight=scenario#scenario) - get_smac_object_callback (Optional[Callable]): Callback function - to create an object of class + get_smac_object_callback (Optional[Callable]): + Callback function to create an object of class [smac.optimizer.smbo.SMBO](https://automl.github.io/SMAC3/master/apidoc/smac.optimizer.smbo.html). The function must accept the arguments scenario_dict, instances, num_params, runhistory, seed and ta. This is an advanced feature. Use only if you are familiar with [SMAC](https://automl.github.io/SMAC3/master/index.html). - all_supported_metrics (bool), (default=True): if True, all - metrics supporting current task will be calculated + all_supported_metrics (bool), (default=True): + if True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results precision (int), (default=32): Numeric precision used when loading ensemble data. Can be either '16', '32' or '64'. disable_file_output (Union[bool, List]): - load_models (bool), (default=True): Whether to load the - models after fitting AutoPyTorch. - + load_models (bool), (default=True): + Whether to load the models after fitting AutoPyTorch. + portfolio_selection (str), (default=None): + This argument controls the initial configurations that + AutoPyTorch uses to warm start SMAC for hyperparameter + optimization. By default, no warm-starting happens. + The user can provide a path to a json file containing + configurations, similar to (...herepathtogreedy...). + Additionally, the keyword 'greedy' is supported, + which would use the default portfolio from + `AutoPyTorch Tabular ` Returns: self @@ -233,6 +245,7 @@ def search( precision=precision, disable_file_output=disable_file_output, load_models=load_models, + portfolio_selection=portfolio_selection, ) def predict( diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index a6f46d2fe..c2dc0eb86 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -47,6 +47,9 @@ class TabularRegressionTask(BaseTask): exclude_components (Optional[Dict]): If None, all possible components are used. Otherwise specifies set of components not to use. Incompatible with include components + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline """ def __init__( @@ -111,6 +114,7 @@ def search( precision: int = 32, disable_file_output: List = [], load_models: bool = True, + portfolio_selection: Optional[str] = None, ) -> 'BaseTask': """ Search for the best pipeline configuration for the given dataset. @@ -175,6 +179,15 @@ def search( disable_file_output (Union[bool, List]): load_models (bool), (default=True): Whether to load the models after fitting AutoPyTorch. + portfolio_selection (str), (default=None): + This argument controls the initial configurations that + AutoPyTorch uses to warm start SMAC for hyperparameter + optimization. By default, no warm-starting happens. + The user can provide a path to a json file containing + configurations, similar to (...herepathtogreedy...). + Additionally, the keyword 'greedy' is supported, + which would use the default portfolio from + `AutoPyTorch Tabular ` Returns: self @@ -221,6 +234,7 @@ def search( precision=precision, disable_file_output=disable_file_output, load_models=load_models, + portfolio_selection=portfolio_selection, ) def predict( diff --git a/autoPyTorch/configs/greedy_portfolio.json b/autoPyTorch/configs/greedy_portfolio.json new file mode 100644 index 000000000..a8e640a4e --- /dev/null +++ b/autoPyTorch/configs/greedy_portfolio.json @@ -0,0 +1,509 @@ +[{"data_loader:batch_size": 60, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "MixUpTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 290, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 6, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.00020060142055000713, + "optimizer:AdamOptimizer:weight_decay": 0.0018320003468984575, + "trainer:MixUpTrainer:alpha": 0.8448753109694546, + "trainer:MixUpTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.023271935735825866}, + {"data_loader:batch_size": 255, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 2, + "network_backbone:ShapedResNetBackbone:max_units": 41, + "network_backbone:ShapedResNetBackbone:num_groups": 4, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.011526647986073339, + "optimizer:AdamOptimizer:weight_decay": 0.031290291410446765, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.30409463597128383, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.7662454727603789}, + {"data_loader:batch_size": 165, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "SGDOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 3, + "network_backbone:ShapedResNetBackbone:max_units": 438, + "network_backbone:ShapedResNetBackbone:num_groups": 1, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:SGDOptimizer:lr": 0.07331878780908542, + "optimizer:SGDOptimizer:momentum": 0.44665514022476815, + "optimizer:SGDOptimizer:weight_decay": 0.006911333726469374, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 299, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 2, + "network_backbone:ShapedResNetBackbone:max_units": 279, + "network_backbone:ShapedResNetBackbone:num_groups": 2, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0007471732018616978, + "optimizer:AdamOptimizer:weight_decay": 0.0005438753720314742, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 183, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 2, + "network_backbone:ShapedResNetBackbone:max_units": 354, + "network_backbone:ShapedResNetBackbone:num_groups": 1, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.001780112494729604, + "optimizer:AdamOptimizer:weight_decay": 0.004224029178574147, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.4412292309825137, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.27204101593048097}, + {"data_loader:batch_size": 21, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 201, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 3, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.015232504956212976, + "optimizer:AdamOptimizer:weight_decay": 9.906036909600088e-05, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 159, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "TruncatedSVD", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:TruncatedSVD:target_dim": 151, + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 966, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 5, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0007790465032701478, + "optimizer:AdamOptimizer:weight_decay": 0.0016722444122252624, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 442, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "TruncatedSVD", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:TruncatedSVD:target_dim": 115, + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 3, + "network_backbone:ShapedResNetBackbone:max_units": 467, + "network_backbone:ShapedResNetBackbone:num_groups": 1, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0008298747674239372, + "optimizer:AdamOptimizer:weight_decay": 0.0067071038164946365, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 140, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "TruncatedSVD", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:TruncatedSVD:target_dim": 240, + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 423, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0006164392898567234, + "optimizer:AdamOptimizer:weight_decay": 0.006605449457495538, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 48, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "SGDOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 529, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 3, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:SGDOptimizer:lr": 0.020107910011636462, + "optimizer:SGDOptimizer:momentum": 0.5818716367708677, + "optimizer:SGDOptimizer:weight_decay": 0.003995594064278902, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 168, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 349, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0019867054473724295, + "optimizer:AdamOptimizer:weight_decay": 0.0067889732830148704, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.8992826006547855}, + {"data_loader:batch_size": 21, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 278, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 3, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.001178107244651597, + "optimizer:AdamOptimizer:weight_decay": 0.010815452216436712, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 163, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 171, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": false, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.002654293880282279, + "optimizer:AdamOptimizer:weight_decay": 0.010374059713414468, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.6341848343636569}, + {"data_loader:batch_size": 150, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 314, + "network_backbone:ShapedResNetBackbone:num_groups": 1, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.003106362796390374, + "optimizer:AdamOptimizer:weight_decay": 0.010492136888557045, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.2808341606307928, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.7133813761319248}, + {"data_loader:batch_size": 151, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "TruncatedSVD", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:TruncatedSVD:target_dim": 147, + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 313, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 3, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0013791902792817097, + "optimizer:AdamOptimizer:weight_decay": 0.0016536079820230513, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128}, + {"data_loader:batch_size": 42, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "TruncatedSVD", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:TruncatedSVD:target_dim": 151, + "lr_scheduler:CosineAnnealingLR:T_max": 50, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 2, + "network_backbone:ShapedResNetBackbone:max_units": 86, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 200, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.0021530021937535334, + "optimizer:AdamOptimizer:weight_decay": 0.008386657635007597, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.034431265307095615, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.6296079567189131}] \ No newline at end of file diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py index ddd6e95a1..2ae894e8b 100644 --- a/autoPyTorch/optimizer/smbo.py +++ b/autoPyTorch/optimizer/smbo.py @@ -4,6 +4,7 @@ import typing import ConfigSpace +from ConfigSpace.configuration_space import Configuration import dask.distributed @@ -25,6 +26,7 @@ ) from autoPyTorch.ensemble.ensemble_builder import EnsembleBuilderManager from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue, get_cost_of_crash +from autoPyTorch.optimizer.utils import read_return_initial_configurations from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates from autoPyTorch.utils.logging_ import get_named_client_logger @@ -40,6 +42,7 @@ def get_smac_object( initial_budget: int, max_budget: int, dask_client: typing.Optional[dask.distributed.Client], + initial_configurations: typing.Optional[typing.List[Configuration]] = None, ) -> SMAC4AC: """ This function returns an SMAC object that is gonna be used as @@ -53,6 +56,8 @@ def get_smac_object( ta_kwargs (typing.Dict[str, typing.Any]): Arguments to the above ta n_jobs (int): Amount of cores to use for this task dask_client (dask.distributed.Client): User provided scheduler + initial_configurations (typing.List[Configuration]): List of initial + configurations which smac will run before starting the search process Returns: (SMAC4AC): sequential model algorithm configuration object @@ -67,7 +72,7 @@ def get_smac_object( runhistory2epm=rh2EPM, tae_runner=ta, tae_runner_kwargs=ta_kwargs, - initial_configurations=None, + initial_configurations=initial_configurations, run_id=seed, intensifier=intensifier, intensifier_kwargs={'initial_budget': initial_budget, 'max_budget': max_budget, @@ -103,7 +108,8 @@ def __init__(self, all_supported_metrics: bool = True, ensemble_callback: typing.Optional[EnsembleBuilderManager] = None, logger_port: typing.Optional[int] = None, - search_space_updates: typing.Optional[HyperparameterSearchSpaceUpdates] = None + search_space_updates: typing.Optional[HyperparameterSearchSpaceUpdates] = None, + portfolio_selection: typing.Optional[str] = None ): """ Interface to SMAC. This method calls the SMAC optimize method, and allows @@ -152,7 +158,15 @@ def __init__(self, Allows to create a user specified SMAC object ensemble_callback (typing.Optional[EnsembleBuilderManager]): A callback used in this scenario to start ensemble building subtasks - + portfolio_selection (str), (default=None): + This argument controls the initial configurations that + AutoPyTorch uses to warm start SMAC for hyperparameter + optimization. By default, no warm-starting happens. + The user can provide a path to a json file containing + configurations, similar to (autoPyTorch/configs/greedy_portfolio.json). + Additionally, the keyword 'greedy' is supported, + which would use the default portfolio from + `AutoPyTorch Tabular ` """ super(AutoMLSMBO, self).__init__() # data related @@ -205,6 +219,11 @@ def __init__(self, port=self.logger_port) self.logger.info("initialised {}".format(self.__class__.__name__)) + self.initial_configurations: typing.Optional[typing.List[Configuration]] = None + if portfolio_selection is not None: + self.initial_configurations = read_return_initial_configurations(config_space=config_space, + portfolio_selection=portfolio_selection) + def reset_data_manager(self) -> None: if self.datamanager is not None: del self.datamanager @@ -314,7 +333,8 @@ def run_smbo(self, func: typing.Optional[typing.Callable] = None n_jobs=self.n_jobs, initial_budget=initial_budget, max_budget=max_budget, - dask_client=self.dask_client) + dask_client=self.dask_client, + initial_configurations=self.initial_configurations) else: smac = get_smac_object(scenario_dict=scenario_dict, seed=seed, @@ -323,7 +343,8 @@ def run_smbo(self, func: typing.Optional[typing.Callable] = None n_jobs=self.n_jobs, initial_budget=initial_budget, max_budget=max_budget, - dask_client=self.dask_client) + dask_client=self.dask_client, + initial_configurations=self.initial_configurations) if self.ensemble_callback is not None: smac.register_callback(self.ensemble_callback) diff --git a/autoPyTorch/optimizer/utils.py b/autoPyTorch/optimizer/utils.py new file mode 100644 index 000000000..6fb9d5024 --- /dev/null +++ b/autoPyTorch/optimizer/utils.py @@ -0,0 +1,33 @@ +import json +import os +import warnings +from typing import Any, Dict, List + +from ConfigSpace.configuration_space import Configuration, ConfigurationSpace + + +def read_return_initial_configurations( + config_space: ConfigurationSpace, + portfolio_selection: str +) -> List[Configuration]: + + # read and validate initial configurations + portfolio_path = portfolio_selection if portfolio_selection != "greedy" else \ + os.path.join(os.path.dirname(__file__), '../configs/greedy_portfolio.json') + try: + initial_configurations_dict: List[Dict[str, Any]] = json.load(open(portfolio_path)) + except FileNotFoundError: + raise FileNotFoundError("The path: {} provided for 'portfolio_selection' for " + "the file containing the portfolio configurations " + "does not exist. Please provide a valid path".format(portfolio_path)) + initial_configurations: List[Configuration] = list() + for configuration_dict in initial_configurations_dict: + try: + configuration = Configuration(config_space, configuration_dict) + initial_configurations.append(configuration) + except Exception as e: + warnings.warn(f"Failed to convert {configuration_dict} into" + f" a Configuration with error {e}. " + f"Therefore, it can't be used as an initial " + f"configuration as it does not match the current config space. ") + return initial_configurations diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index fc086c902..4eb6a5213 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -41,6 +41,9 @@ class BasePipeline(Pipeline): random_state (np.random.RandomState): allows to produce reproducible results by setting a seed for randomized settings init_params (Optional[Dict[str, Any]]) + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline Attributes: diff --git a/autoPyTorch/pipeline/image_classification.py b/autoPyTorch/pipeline/image_classification.py index b31c8dbf2..bf15d738a 100644 --- a/autoPyTorch/pipeline/image_classification.py +++ b/autoPyTorch/pipeline/image_classification.py @@ -40,7 +40,18 @@ class ImageClassificationPipeline(ClassifierMixin, BasePipeline): Args: config (Configuration) The configuration to evaluate. - random_state (Optional[RandomState): random_state is the random number generator + steps (Optional[List[Tuple[str, autoPyTorchChoice]]]): the list of steps that + build the pipeline. If provided, they won't be dynamically produced. + include (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to honor during the creation of the configuration space. + exclude (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to avoid during the creation of the configuration space. + random_state (np.random.RandomState): allows to produce reproducible results by + setting a seed for randomized settings + init_params (Optional[Dict[str, Any]]) + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline Attributes: Examples diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index bb4cb10ac..ef57a8569 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -60,7 +60,18 @@ class TabularClassificationPipeline(ClassifierMixin, BasePipeline): Args: config (Configuration) The configuration to evaluate. - random_state (Optional[RandomState): random_state is the random number generator + steps (Optional[List[Tuple[str, autoPyTorchChoice]]]): the list of steps that + build the pipeline. If provided, they won't be dynamically produced. + include (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to honor during the creation of the configuration space. + exclude (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to avoid during the creation of the configuration space. + random_state (np.random.RandomState): allows to produce reproducible results by + setting a seed for randomized settings + init_params (Optional[Dict[str, Any]]) + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline Attributes: Examples diff --git a/autoPyTorch/pipeline/tabular_regression.py b/autoPyTorch/pipeline/tabular_regression.py index af8702695..2650868b6 100644 --- a/autoPyTorch/pipeline/tabular_regression.py +++ b/autoPyTorch/pipeline/tabular_regression.py @@ -58,7 +58,18 @@ class TabularRegressionPipeline(RegressorMixin, BasePipeline): Args: config (Configuration) The configuration to evaluate. - random_state (Optional[RandomState): random_state is the random number generator + steps (Optional[List[Tuple[str, autoPyTorchChoice]]]): the list of steps that + build the pipeline. If provided, they won't be dynamically produced. + include (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to honor during the creation of the configuration space. + exclude (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to avoid during the creation of the configuration space. + random_state (np.random.RandomState): allows to produce reproducible results by + setting a seed for randomized settings + init_params (Optional[Dict[str, Any]]) + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline Attributes: Examples diff --git a/autoPyTorch/pipeline/traditional_tabular_classification.py b/autoPyTorch/pipeline/traditional_tabular_classification.py index 51d8e6616..49be2a1fa 100644 --- a/autoPyTorch/pipeline/traditional_tabular_classification.py +++ b/autoPyTorch/pipeline/traditional_tabular_classification.py @@ -10,6 +10,7 @@ from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice from autoPyTorch.pipeline.components.setup.traditional_ml.base_model_choice import ModelChoice +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates class TraditionalTabularClassificationPipeline(ClassifierMixin, BasePipeline): @@ -19,7 +20,19 @@ class TraditionalTabularClassificationPipeline(ClassifierMixin, BasePipeline): Args: config (Configuration) The configuration to evaluate. - random_state (Optional[RandomState): random_state is the random number generator + steps (Optional[List[Tuple[str, autoPyTorchChoice]]]): the list of steps that + build the pipeline. If provided, they won't be dynamically produced. + include (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to honor during the creation of the configuration space. + exclude (Optional[Dict[str, Any]]): Allows the caller to specify which configurations + to avoid during the creation of the configuration space. + random_state (np.random.RandomState): allows to produce reproducible results by + setting a seed for randomized settings + init_params (Optional[Dict[str, Any]]) + search_space_updates (Optional[HyperparameterSearchSpaceUpdates]): + search space updates that can be used to modify the search + space of particular components or choice modules of the pipeline + Attributes: """ @@ -32,11 +45,12 @@ def __init__( include: Optional[Dict[str, Any]] = None, exclude: Optional[Dict[str, Any]] = None, random_state: Optional[np.random.RandomState] = None, - init_params: Optional[Dict[str, Any]] = None + init_params: Optional[Dict[str, Any]] = None, + search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None ): super().__init__( config, steps, dataset_properties, include, exclude, - random_state, init_params) + random_state, init_params, search_space_updates) def predict(self, X: np.ndarray, batch_size: Optional[int] = None ) -> np.ndarray: diff --git a/examples/40_advanced/example_run_with_portfolio.py b/examples/40_advanced/example_run_with_portfolio.py new file mode 100644 index 000000000..4109a2378 --- /dev/null +++ b/examples/40_advanced/example_run_with_portfolio.py @@ -0,0 +1,71 @@ +""" +============================================ +Tabular Classification with Greedy Portfolio +============================================ + +The following example shows how to fit a sample classification model +with AutoPyTorch using the greedy portfolio +""" +import os +import tempfile as tmp +import warnings + +os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() +os.environ['OMP_NUM_THREADS'] = '1' +os.environ['OPENBLAS_NUM_THREADS'] = '1' +os.environ['MKL_NUM_THREADS'] = '1' + +warnings.simplefilter(action='ignore', category=UserWarning) +warnings.simplefilter(action='ignore', category=FutureWarning) + +import sklearn.datasets +import sklearn.model_selection + +from autoPyTorch.api.tabular_classification import TabularClassificationTask + + +if __name__ == '__main__': + + ############################################################################ + # Data Loading + # ============ + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=42, + ) + + ############################################################################ + # Build and fit a classifier + # ========================== + api = TabularClassificationTask( + seed=42, + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50, + # Setting this option to "greedy" + # will make smac run the configurations + # present in 'autoPyTorch/configs/greedy_portfolio.json' + portfolio_selection="greedy" + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + # Print the final ensemble built by AutoPyTorch + print(api.show_models()) diff --git a/setup.py b/setup.py index 4fd732fdd..a8522a8dc 100755 --- a/setup.py +++ b/setup.py @@ -11,10 +11,10 @@ # noinspection PyInterpreter setuptools.setup( name="autoPyTorch", - version="0.0.3", + version="0.1.0", author="AutoML Freiburg", author_email="zimmerl@informatik.uni-freiburg.de", - description=("Auto-PyTorch searches neural architectures using BO-HB"), + description=("Auto-PyTorch searches neural architectures using smac"), long_description=long_description, url="https://github.com/automl/Auto-PyTorch", long_description_content_type="text/markdown", @@ -59,5 +59,6 @@ "docs": ["sphinx", "sphinx-gallery", "sphinx_bootstrap_theme", "numpydoc"], }, test_suite="pytest", - data_files=[('configs', ['autoPyTorch/configs/default_pipeline_options.json'])] + data_files=[('configs', ['autoPyTorch/configs/default_pipeline_options.json']), + ('portfolio', ['autoPyTorch/configs/greedy_portfolio.json'])] ) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 280617306..9f9d9f765 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -1,3 +1,4 @@ +import json import os import pathlib import pickle @@ -464,3 +465,92 @@ def test_do_dummy_prediction(dask_client, fit_dictionary_tabular): estimator._clean_logger() del estimator + + +@unittest.mock.patch('autoPyTorch.evaluation.train_evaluator.eval_function', + new=dummy_eval_function) +@pytest.mark.parametrize('openml_id', (40981, )) +def test_portfolio_selection(openml_id, backend, n_samples): + + # Get the data and check that contents of data-manager make sense + X, y = sklearn.datasets.fetch_openml( + data_id=int(openml_id), + return_X_y=True, as_frame=True + ) + X, y = X.iloc[:n_samples], y.iloc[:n_samples] + + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, y, random_state=1) + + include = None + # for python less than 3.7, learned entity embedding + # is not able to be stored on disk (only on CI) + if sys.version_info < (3, 7): + include = {'network_embedding': ['NoEmbedding']} + # Search for a good configuration + estimator = TabularClassificationTask( + backend=backend, + resampling_strategy=HoldoutValTypes.holdout_validation, + include_components=include + ) + + with unittest.mock.patch.object(estimator, '_do_dummy_prediction', new=dummy_do_dummy_prediction): + estimator.search( + X_train=X_train, y_train=y_train, + X_test=X_test, y_test=y_test, + optimize_metric='accuracy', + total_walltime_limit=30, + func_eval_time_limit_secs=5, + enable_traditional_pipeline=False, + portfolio_selection=os.path.join(os.path.dirname(__file__), + "../../autoPyTorch/configs/greedy_portfolio.json") + ) + + successful_config_ids = [run_key.config_id for run_key, run_value in estimator.run_history.data.items( + ) if 'SUCCESS' in str(run_value.status)] + successful_configs = [estimator.run_history.ids_config[id].get_dictionary() for id in successful_config_ids] + portfolio_configs = json.load(open(os.path.join(os.path.dirname(__file__), + "../../autoPyTorch/configs/greedy_portfolio.json"))) + # check if any configs from greedy portfolio were compatible with australian + assert any(successful_config in portfolio_configs for successful_config in successful_configs) + + +@unittest.mock.patch('autoPyTorch.evaluation.train_evaluator.eval_function', + new=dummy_eval_function) +@pytest.mark.parametrize('openml_id', (40981, )) +def test_portfolio_selection_failure(openml_id, backend, n_samples): + + # Get the data and check that contents of data-manager make sense + X, y = sklearn.datasets.fetch_openml( + data_id=int(openml_id), + return_X_y=True, as_frame=True + ) + X, y = X.iloc[:n_samples], y.iloc[:n_samples] + + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, y, random_state=1) + + include = None + # for python less than 3.7, learned entity embedding + # is not able to be stored on disk (only on CI) + if sys.version_info < (3, 7): + include = {'network_embedding': ['NoEmbedding']} + # Search for a good configuration + estimator = TabularClassificationTask( + backend=backend, + resampling_strategy=HoldoutValTypes.holdout_validation, + include_components=include + ) + + with pytest.raises(FileNotFoundError, match=r"The path: .+? provided for 'portfolio_selection' " + r"for the file containing the portfolio configurations " + r"does not exist\. Please provide a valid path"): + estimator.search( + X_train=X_train, y_train=y_train, + X_test=X_test, y_test=y_test, + optimize_metric='accuracy', + total_walltime_limit=30, + func_eval_time_limit_secs=5, + enable_traditional_pipeline=False, + portfolio_selection="random_path_to_test.json" + ) diff --git a/test/test_pipeline/components/training/base.py b/test/test_pipeline/components/training/base.py index d7cb2ebd1..38b6b5007 100644 --- a/test/test_pipeline/components/training/base.py +++ b/test/test_pipeline/components/training/base.py @@ -23,6 +23,8 @@ def prepare_trainer(self, trainer: BaseTrainerComponent, task_type: int, epochs=50): + torch.manual_seed(1) + if task_type in CLASSIFICATION_TASKS: X, y = make_classification( n_samples=n_samples, diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 98ea47716..36670e325 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -33,6 +33,7 @@ OVERFIT_EPOCHS = 1000 +N_SAMPLES = 500 class BaseDataLoaderTest(unittest.TestCase): @@ -127,7 +128,7 @@ def test_fit_transform(self): class TestBaseTrainerComponent(BaseTraining): - def test_evaluate(self, n_samples): + def test_evaluate(self): """ Makes sure we properly evaluate data, returning a proper loss and metric @@ -139,7 +140,7 @@ def test_evaluate(self, n_samples): loader, criterion, epochs, - logger) = self.prepare_trainer(n_samples, + logger) = self.prepare_trainer(N_SAMPLES, BaseTrainerComponent(), constants.TABULAR_CLASSIFICATION)