Config API + Optuna PoC = AutoML 🖤

.github/workflows/dl_cpu_core_tune.yml

@@ -0,0 +1,88 @@
+name: catalyst-tune
+# <- standard block end ->
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - develop
+      - master
+
+
+jobs:
+  build:
+    name: dl-cpu-core-tune
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      # max-parallel: 6
+      matrix:
+        os: [ubuntu-18.04]  # macOS-10.15, windows-2019,
+        python-version: [3.8]  # 3.6, 3.7,
+        requirements: ['latest']  # 'minimal',
+        exclude:
+          # pypi problems
+          - python-version: 3.8
+            requirements: 'minimal'
+          - python-version: 3.7
+            requirements: 'minimal'
+          # pickle problems
+          - python-version: 3.8
+            os: macOS-10.15
+          - python-version: 3.6
+            os: macOS-10.15
+
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # https://github.com/pytorch/pytorch/issues/20030
+      - name: Setup macOS
+        if: startsWith(runner.os, 'macOS')
+        run: |
+          brew install libomp
+          brew install gnu-sed
+
+      - name: set minimal dependencies
+        if: matrix.requirements == 'minimal'
+        run: |
+          python -c "req = open('./requirements/requirements.txt').read().replace('>', '=') ; open('./requirements/requirements.txt', 'w').write(req)"
+          python -c "req = open('./requirements/requirements-cv.txt').read().replace('>', '=') ; open('./requirements/requirements-cv.txt', 'w').write(req)"
+          python -c "req = open('./requirements/requirements-nlp.txt').read().replace('>', '=') ; open('./requirements/requirements-nlp.txt', 'w').write(req)"
+
+      # https://github.com/actions/cache/blob/master/examples.md
+      # Note: This uses an internal pip API and may not always work
+      # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
+      - name: get pip cache dir
+        id: pip-cache
+        run: |
+          python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
+
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requirements }}-pip-${{ hashFiles('./requirements/requirements.txt') }}-${{ hashFiles('./tests/requirements.txt') }}-${{ hashFiles('./requirements/requirements-cv.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requirements }}-pip-
+
+      - name: install dependencies
+        run: |
+          # python -m pip install --upgrade --user pip
+          pip install -r ./requirements/requirements.txt -r ./tests/requirements.txt  -r ./requirements/requirements-cv.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade-strategy only-if-needed
+          python --version
+          pip --version
+          pip list
+        shell: bash
+# <- standard block end ->
+
+      - name: check examples
+        env:
+          REQUIREMENTS: ${{ matrix.requirements }}
+        run: bash ./bin/tests/check_core_tune.sh

CHANGELOG.md

@@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 ### Added
 
 - Runner registry support for Config API ([#936](https://github.com/catalyst-team/catalyst/pull/936))
+- `catalyst-dl tune` command - Optuna with Config API integration for AutoML hyperparameters optimization ([#937](https://github.com/catalyst-team/catalyst/pull/937))
+- `OptunaPruningCallback` alias for `OptunaCallback` ([#937](https://github.com/catalyst-team/catalyst/pull/937))
 - AdamP and SGDP to `catalyst.contrib.nn.criterion` ([#942](https://github.com/catalyst-team/catalyst/pull/942))
 
 ### Changed

bin/tests/check_core_tune.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails
+set -eo pipefail -v
+
+pip install -r requirements/requirements.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade-strategy only-if-needed
+
+################################  pipeline 01  ################################
+echo 'pipeline 01'
+EXPDIR=./examples/cifar_stages_optuna
+BASELOGDIR=./examples/logs/cifar_stages_optuna
+
+PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+  python catalyst/dl/scripts/tune.py \
+  --expdir=${EXPDIR} \
+  --config=${EXPDIR}/config.yml \
+  --baselogdir=${BASELOGDIR} \
+  --n-trials=3 \
+  --timeout=600
+
+if [[ ! -d "$BASELOGDIR" ]]; then
+  echo "Directory $BASELOGDIR does not exist"
+  exit 1
+fi

catalyst/contrib/dl/callbacks/__init__.py

@@ -93,7 +93,10 @@
 
 try:
     import optuna
-    from catalyst.contrib.dl.callbacks.optuna_callback import OptunaCallback
+    from catalyst.contrib.dl.callbacks.optuna_callback import (
+        OptunaPruningCallback,
+        OptunaCallback,
+    )
 except ImportError as ex:
     if settings.optuna_required:
         logger.warning(

catalyst/contrib/dl/callbacks/optuna_callback.py

@@ -3,7 +3,7 @@
 from catalyst.core import Callback, CallbackOrder, IRunner
 
 
-class OptunaCallback(Callback):
+class OptunaPruningCallback(Callback):
     """
     Optuna callback for pruning unpromising runs
 
@@ -25,7 +25,7 @@ def objective(trial: optuna.Trial):
                 criterion=criterion,
                 optimizer=optimizer,
                 callbacks=[
-                    OptunaCallback(trial)
+                    OptunaPruningCallback(trial)
                     # some other callbacks ...
                 ],
                 num_epochs=num_epochs,
@@ -35,34 +35,61 @@ def objective(trial: optuna.Trial):
         study = optuna.create_study()
         study.optimize(objective, n_trials=100, timeout=600)
 
-    Config API is not supported.
+    Config API is supported through `catalyst-dl tune` command.
     """
 
-    def __init__(self, trial: optuna.Trial):
+    def __init__(self, trial: optuna.Trial = None):
         """
         This callback can be used for early stopping (pruning)
         unpromising runs.
 
         Args:
              trial: Optuna.Trial for experiment.
         """
-        super(OptunaCallback, self).__init__(CallbackOrder.External)
+        super().__init__(CallbackOrder.External)
         self.trial = trial
 
-    def on_epoch_end(self, runner: "IRunner"):
+    def on_stage_start(self, runner: "IRunner"):
         """
-        On epoch end action.
+        On stage start hook.
+        Takes ``optuna_trial`` from ``Experiment`` for future usage if needed.
 
-        Considering prune or not to prune current run at current epoch.
+        Args:
+            runner: runner for current experiment
 
         Raises:
-            TrialPruned: if current run should be pruned
+            NotImplementedError: if no Optuna trial was found on stage start.
+        """
+        trial = runner.experiment.trial
+        if (
+            self.trial is None
+            and trial is not None
+            and isinstance(trial, optuna.Trial)
+        ):
+            self.trial = trial
+
+        if self.trial is None:
+            raise NotImplementedError("No Optuna trial found for logging")
+
+    def on_epoch_end(self, runner: "IRunner"):
+        """
+        On epoch end hook.
+
+        Considering prune or not to prune current run at current epoch.
 
         Args:
             runner: runner for current experiment
+
+        Raises:
+            TrialPruned: if current run should be pruned
         """
         metric_value = runner.valid_metrics[runner.main_metric]
         self.trial.report(metric_value, step=runner.epoch)
         if self.trial.should_prune():
             message = "Trial was pruned at epoch {}.".format(runner.epoch)
             raise optuna.TrialPruned(message)
+
+
+OptunaCallback = OptunaPruningCallback
+
+__all__ = ["OptunaCallback", "OptunaPruningCallback"]

catalyst/contrib/dl/callbacks/tests/test_optuna_callback.py

@@ -7,7 +7,7 @@
 
 from catalyst import dl
 from catalyst.contrib.datasets import MNIST
-from catalyst.contrib.dl.callbacks import OptunaCallback
+from catalyst.contrib.dl.callbacks import OptunaPruningCallback
 from catalyst.contrib.nn import Flatten
 from catalyst.data.cv.transforms.torch import ToTensor
 from catalyst.dl import AccuracyCallback
@@ -39,7 +39,7 @@ def objective(trial):
             criterion=criterion,
             optimizer=optimizer,
             callbacks=[
-                OptunaCallback(trial),
+                OptunaPruningCallback(trial),
                 AccuracyCallback(num_classes=10),
             ],
             num_epochs=10,

catalyst/core/experiment.py

@@ -63,7 +63,8 @@ def logdir(self) -> str:
     @property
     @abstractmethod
     def hparams(self) -> OrderedDict:
-        """Returns hyper-parameters
+        """
+        Returns hyper-parameters for current experiment.
 
         Example::
             >>> experiment.hparams
@@ -78,17 +79,16 @@ def hparams(self) -> OrderedDict:
 
     @property
     @abstractmethod
-    def stages(self) -> Iterable[str]:
-        """Experiment's stage names.
+    def trial(self) -> Any:
+        """
+        Returns hyperparameter trial for current experiment.
+        Could be usefull for Optuna/HyperOpt/Ray.tune
+        hyperparameters optimizers.
 
         Example::
 
-            >>> experiment.stages
-            ["pretraining", "training", "finetuning"]
-
-        .. note::
-            To understand stages concept, please follow Catalyst documentation,
-            for example, :py:mod:`catalyst.core.callback.Callback`
+            >>> experiment.trial
+            optuna.trial._trial.Trial  # Optuna variant
         """
         pass
 
@@ -113,6 +113,22 @@ def distributed_params(self) -> Dict:
         """
         pass
 
+    @property
+    @abstractmethod
+    def stages(self) -> Iterable[str]:
+        """Experiment's stage names.
+
+        Example::
+
+            >>> experiment.stages
+            ["pretraining", "training", "finetuning"]
+
+        .. note::
+            To understand stages concept, please follow Catalyst documentation,
+            for example, :py:mod:`catalyst.core.callback.Callback`
+        """
+        pass
+
     @abstractmethod
     def get_stage_params(self, stage: str) -> Mapping[str, Any]:
         """Returns extra stage parameters for a given stage.

catalyst/dl/__main__.py

@@ -3,7 +3,7 @@
 
 from catalyst.__version__ import __version__
 from catalyst.dl.scripts import quantize, run, trace
-from catalyst.tools.settings import IS_GIT_AVAILABLE
+from catalyst.tools.settings import IS_GIT_AVAILABLE, IS_OPTUNA_AVAILABLE
 
 COMMANDS = OrderedDict(
     [("run", run), ("trace", trace), ("quantize", quantize)]
@@ -12,6 +12,10 @@
     from catalyst.dl.scripts import init
 
     COMMANDS["init"] = init
+if IS_OPTUNA_AVAILABLE:
+    from catalyst.dl.scripts import tune
+
+    COMMANDS["tune"] = tune
 
 
 def build_parser() -> ArgumentParser: