[Refactor] Use the backend implementation from automl common (automl#185

)

* [ADD] First push to enable common backend

* Fix unit test

* Try public https

* [FIX] conftest prefix

* [fix] unit test

* [FIX] Fix fixture in score

* [Fix] pytest collection

* [FIX] flake

* [FIX] regression also!

* Update README.md

Co-authored-by: Ravin Kohli <[email protected]>

* Update .gitmodules

Co-authored-by: Ravin Kohli <[email protected]>

* [FIX] Regression time

* Make flaky in case memout doesn't happen

* Refacto development automl common backend debug (automl#2)

* [ADD] debug information

* [FIX] try fork for more stability

Co-authored-by: Ravin Kohli <[email protected]>

.github/workflows/docs.yml

@@ -12,6 +12,7 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
+        git submodule update --init --recursive
         pip install -e .[docs,examples]
     - name: Make docs
       run: |

.github/workflows/examples.yml

@@ -20,6 +20,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install test dependencies
       run: |
+        git submodule update --init --recursive
         python -m pip install --upgrade pip
         pip install -e .[examples]
         which python

.github/workflows/pre-commit.yaml

@@ -11,6 +11,9 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
+    - name: Init Submodules
+      run: |
+        git submodule update --init --recursive
     - name: Install pre-commit
       run: |
         pip install pre-commit

.github/workflows/pytest.yml

@@ -20,6 +20,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install test dependencies
       run: |
+        git submodule update --init --recursive
         python -m pip install --upgrade pip
         pip install -e .[test]
     - name: Store repository status

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "autoPyTorch/automl_common"]
+	path = autoPyTorch/automl_common
+	url = https://github.com/automl/automl_common.git

README.md

@@ -12,20 +12,34 @@ Find the documentation [here](https://automl.github.io/Auto-PyTorch/development)
 
 ## Installation
 
-### Pip
+### Manual Installation
 
 We recommend using Anaconda for developing as follows:
 
 ```sh
 # Following commands assume the user is in a cloned directory of Auto-Pytorch
+
+# We also need to initialize the automl_common repository as follows
+# You can find more information about this here:
+# https://github.com/automl/automl_common/
+git submodule update --init --recursive
+
+# Create the environment
 conda create -n autopytorch python=3.8
 conda activate autopytorch
-conda install gxx_linux-64 gcc_linux-64 swig
+For Linux:
+    conda install gxx_linux-64 gcc_linux-64 swig
+For mac:
+    conda install -c conda-forge clang_osx-64 clangxx_osx-64
+    conda install -c anaconda swig
 cat requirements.txt | xargs -n 1 -L 1 pip install
 python setup.py install
 
 ```
 
+### Pip
+TODO
+
 ## Contributing
 
 If you want to contribute to Auto-PyTorch, clone the repository and checkout our current development branch

autoPyTorch/api/base_task.py

@@ -28,6 +28,7 @@
 from smac.stats.stats import Stats
 from smac.tae import StatusType
 
+from autoPyTorch.automl_common.common.utils.backend import Backend, create
 from autoPyTorch.constants import (
     REGRESSION_TASKS,
     STRING_TO_OUTPUT_TYPES,
@@ -45,7 +46,6 @@
 from autoPyTorch.pipeline.components.setup.traditional_ml.classifier_models import get_available_classifiers
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
 from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score, get_metrics
-from autoPyTorch.utils.backend import Backend, create
 from autoPyTorch.utils.common import FitRequirement, replace_string_bool_to_bool
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.utils.logging_ import (
@@ -157,6 +157,7 @@ def __init__(
             self._backend = backend
         else:
             self._backend = create(
+                prefix='autoPyTorch',
                 temporary_directory=self._temporary_directory,
                 output_directory=self._output_directory,
                 delete_tmp_folder_after_terminate=delete_tmp_folder_after_terminate,
@@ -776,7 +777,6 @@ def _search(
             self
 
         """
-
         if self.task_type != dataset.task_type:
             raise ValueError("Incompatible dataset entered for current task,"
                              "expected dataset to have task type :{} got "
@@ -794,6 +794,10 @@ def _search(
 
         if self._logger is None:
             self._logger = self._get_logger(self.dataset_name)
+
+        # Setup the logger for the backend
+        self._backend.setup_logger(port=self._logger_port)
+
         self._all_supported_metrics = all_supported_metrics
         self._disable_file_output = disable_file_output
         self._memory_limit = memory_limit

autoPyTorch/api/tabular_classification.py

@@ -7,6 +7,7 @@
 import pandas as pd
 
 from autoPyTorch.api.base_task import BaseTask
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import (
     TABULAR_CLASSIFICATION,
     TASK_TYPES_TO_STRING,
@@ -19,7 +20,6 @@
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 
 

autoPyTorch/api/tabular_regression.py

@@ -7,6 +7,7 @@
 import pandas as pd
 
 from autoPyTorch.api.base_task import BaseTask
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import (
     TABULAR_REGRESSION,
     TASK_TYPES_TO_STRING
@@ -19,7 +20,6 @@
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 
 

autoPyTorch/ensemble/ensemble_builder.py

@@ -29,12 +29,12 @@
 from smac.optimizer.smbo import SMBO
 from smac.runhistory.runhistory import RunInfo, RunValue
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import BINARY
 from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble
 from autoPyTorch.ensemble.ensemble_selection import EnsembleSelection
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
 from autoPyTorch.pipeline.components.training.metrics.utils import calculate_loss, calculate_score
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.logging_ import get_named_client_logger
 
 Y_ENSEMBLE = 0

autoPyTorch/ensemble/singlebest_ensemble.py

@@ -5,10 +5,10 @@
 
 from smac.runhistory.runhistory import RunHistory
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble
 from autoPyTorch.pipeline.base_pipeline import BasePipeline
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
-from autoPyTorch.utils.backend import Backend
 
 
 class SingleBest(AbstractEnsemble):

autoPyTorch/evaluation/abstract_evaluator.py

@@ -20,6 +20,7 @@
 import autoPyTorch.pipeline.tabular_classification
 import autoPyTorch.pipeline.tabular_regression
 import autoPyTorch.pipeline.traditional_tabular_classification
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import (
     CLASSIFICATION_TASKS,
     IMAGE_TASKS,
@@ -41,7 +42,6 @@
     calculate_loss,
     get_metrics,
 )
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.common import subsampler
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger
@@ -399,7 +399,6 @@ def __init__(self, backend: Backend,
             name=logger_name,
             port=logger_port,
         )
-        self.backend.setup_logger(name=logger_name, port=logger_port)
 
         self.Y_optimization: Optional[np.ndarray] = None
         self.Y_actual_train: Optional[np.ndarray] = None

autoPyTorch/evaluation/tae.py

@@ -23,9 +23,9 @@
 from smac.tae.execute_func import AbstractTAFunc
 
 import autoPyTorch.evaluation.train_evaluator
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.evaluation.utils import empty_queue, extract_learning_curve, read_queue
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.common import replace_string_bool_to_bool
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger

autoPyTorch/evaluation/train_evaluator.py

@@ -9,6 +9,7 @@
 
 from smac.tae import StatusType
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.constants import (
     CLASSIFICATION_TASKS,
     MULTICLASSMULTIOUTPUT,
@@ -18,7 +19,6 @@
     fit_and_suppress_warnings
 )
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.common import subsampler
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 

autoPyTorch/optimizer/smbo.py

@@ -16,6 +16,7 @@
 from smac.tae.serial_runner import SerialRunner
 from smac.utils.io.traj_logging import TrajEntry
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
@@ -25,7 +26,6 @@
 from autoPyTorch.ensemble.ensemble_builder import EnsembleBuilderManager
 from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue, get_cost_of_crash
 from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.utils.logging_ import get_named_client_logger
 from autoPyTorch.utils.stopwatch import StopWatch

autoPyTorch/pipeline/components/preprocessing/base_preprocessing.py

@@ -10,8 +10,8 @@
 
 import torch
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.common import FitRequirement
 
 

autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py

@@ -11,9 +11,9 @@
 
 import torchvision
 
+from autoPyTorch.automl_common.common.utils.backend import Backend
 from autoPyTorch.datasets.base_dataset import BaseDataset
 from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent
-from autoPyTorch.utils.backend import Backend
 from autoPyTorch.utils.common import (
     FitRequirement,
     HyperparameterSearchSpace,