Arch update - 20.10

CHANGELOG.md

@@ -22,7 +22,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - `IRunner._prepare_for_stage` logic moved to `IStageBasedRunner.prepare_for_stage` ([#947](https://github.com/catalyst-team/catalyst/pull/947))
     - now we create components in the following order: datasets/loaders, model, criterion, optimizer, scheduler, callbacks
 - `MnistMLDataset` and `MnistQGDataset` data split logic - now targets of the datasets are disjoint ([#949](https://github.com/catalyst-team/catalyst/pull/949))
-- 
+- architecture redesign ([#953](https://github.com/catalyst-team/catalyst/pull/953))
+    - experiments, runners, callbacks grouped by primitives under `catalyst.experiments`/`catalyst.runners`/`catalyst.callbacks` respectively
+    - settings and typing moved from `catalyst.tools.*` to `catalyst.*`
+    - utils moved from `catalyst.*.utils` to `catalyst.utils`
 
 ### Removed
 

bin/tests/check_core_settings.sh

@@ -21,7 +21,7 @@ nlp_required = false
 EOT
 
 python -c """
-from catalyst.contrib.dl import callbacks
+from catalyst.contrib import callbacks
 from catalyst.contrib import utils
 
 try:
@@ -45,12 +45,12 @@ EOT
 
 # check if fail if requirements not installed
 python -c """
-from catalyst.tools import settings
+from catalyst.settings import SETTINGS
 
-assert settings.use_lz4 == False and settings.use_pyarrow == False
+assert SETTINGS.use_lz4 == False and SETTINGS.use_pyarrow == False
 
 try:
-    from catalyst.contrib.dl.callbacks import AlchemyLogger, VisdomLogger
+    from catalyst.contrib.callbacks import AlchemyLogger
 except ImportError:
     pass  # Ok
 else:
@@ -61,7 +61,7 @@ pip install -r requirements/requirements-contrib.txt --quiet --find-links https:
 pip install -r requirements/requirements-ecosystem.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade-strategy only-if-needed
 
 python -c """
-from catalyst.contrib.dl.callbacks import AlchemyLogger, VisdomLogger
+from catalyst.contrib.callbacks import AlchemyLogger
 """
 
 
@@ -77,13 +77,13 @@ EOT
 
 # check if fail if requirements not installed
 python -c """
-from catalyst.tools import settings
+from catalyst.settings import SETTINGS
 
-assert settings.use_libjpeg_turbo == False
+assert SETTINGS.use_libjpeg_turbo == False
 
 try:
     from catalyst.contrib.data import cv as cv_data
-    from catalyst.contrib.dl.callbacks import InferMaskCallback
+    from catalyst.contrib.callbacks import InferMaskCallback
     from catalyst.contrib.models import cv as cv_models
     from catalyst.contrib.utils import imread, imwrite
     from catalyst.data.__main__ import COMMANDS
@@ -103,7 +103,7 @@ pip install -r requirements/requirements-cv.txt --quiet --find-links https://dow
 
 python -c """
 from catalyst.contrib.data import cv as cv_data
-from catalyst.contrib.dl.callbacks import InferMaskCallback
+from catalyst.contrib.callbacks import InferMaskCallback
 from catalyst.contrib.models import cv as cv_models
 from catalyst.contrib.utils import imread, imwrite
 from catalyst.data.__main__ import COMMANDS

bin/tests/check_core_tune.sh

@@ -10,7 +10,7 @@ echo 'pipeline 01'
 EXPDIR=./examples/cifar_stages_optuna
 BASELOGDIR=./examples/logs/cifar_stages_optuna
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/tune.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config.yml \

bin/tests/check_dl_core_.sh

@@ -9,8 +9,8 @@ rm -rf ./tests/logs
 
 
 ################################  pipeline 01  ################################
-(set -e; for f in tests/_tests_scripts/core_*.py; do PYTHONPATH=./catalyst:${PYTHONPATH} python "$f"; done)
-(set -e; for f in tests/_tests_scripts/dl_*.py; do PYTHONPATH=./catalyst:${PYTHONPATH} python "$f"; done)
+(set -e; for f in tests/_tests_scripts/core_*.py; do PYTHONPATH=.:${PYTHONPATH} python "$f"; done)
+(set -e; for f in tests/_tests_scripts/dl_*.py; do PYTHONPATH=.:${PYTHONPATH} python "$f"; done)
 
 
 ################################  pipeline 99  ################################

bin/tests/check_dl_core_callbacks.sh

@@ -63,7 +63,7 @@ function check_line_counts {
 LOG_MSG='pipeline 00'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -83,7 +83,7 @@ check_num_files ${CHECKPOINTS} 7   # 3x2 checkpoints + metrics.json
 
 ############################# CHECK QUANTIZATION FEATURE ####################################
 
-quantization=$(python -c "from catalyst.tools.settings import IS_QUANTIZATION_AVAILABLE; print(int(IS_QUANTIZATION_AVAILABLE))")
+quantization=$(python -c "from catalyst.settings import IS_QUANTIZATION_AVAILABLE; print(int(IS_QUANTIZATION_AVAILABLE))")
 
 echo "pytorch quantization available: ${quantization}"
 
@@ -101,7 +101,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 01'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config1.yml \
@@ -127,7 +127,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 02'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config2.yml \
@@ -156,7 +156,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 03'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config3.yml \
@@ -182,7 +182,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 04'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config4.yml \
@@ -211,7 +211,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 05'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config5.yml \
@@ -238,7 +238,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 06'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config6.yml \
@@ -264,7 +264,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 07'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config7.yml \
@@ -290,7 +290,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 08'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config8.yml \
@@ -321,7 +321,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 09'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config9.yml \
@@ -350,7 +350,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 10'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config10.yml \
@@ -379,7 +379,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 11'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config11.yml \
@@ -410,7 +410,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/for_resume
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -432,7 +432,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/resumed
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config12.yml \
@@ -462,7 +462,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/for_resume
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -484,7 +484,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/resumed
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config13.yml \
@@ -513,7 +513,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config14.yml \
@@ -542,7 +542,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 15'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config15.yml \
@@ -573,7 +573,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 16'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config16.yml \
@@ -604,7 +604,7 @@ rm -rf ${LOGDIR} ${EXP_OUTPUT}
 LOG_MSG='pipeline 17'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config17.yml \
@@ -636,7 +636,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/for_resume
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -658,7 +658,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/other
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config18.yml \
@@ -690,7 +690,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/for_resume
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -712,7 +712,7 @@ LOGDIR=./tests/logs/_tests_dl_callbacks/other
 CHECKPOINTS=${LOGDIR}/checkpoints
 LOGFILE=${CHECKPOINTS}/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config19.yml \
@@ -748,7 +748,7 @@ rm -rf ./tests/logs/_tests_dl_callbacks ${EXP_OUTPUT}
 #TRACE=${LOGDIR}/trace
 #LOGFILE=${CHECKPOINTS}/_metrics.json
 #
-#PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+#PYTHONPATH=./examples:.:${PYTHONPATH} \
 #  python catalyst/dl/scripts/run.py \
 #  --expdir=${EXPDIR} \
 #  --config=${EXPDIR}/config20.yml \
@@ -778,7 +778,7 @@ EXPDIR=./tests/_tests_ml_cmcscore
 LOGDIR=./tests/logs/_tests_ml_cmcscore
 LOGFILE=${LOGDIR}/checkpoints/_metrics.json
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config1.yml \
@@ -814,15 +814,15 @@ LOGFILE=${CHECKPOINTS}/_metrics.json
 EXP_OUTPUT=./tests/output.txt
 
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --stages/stage1/stage_params/num_epochs='2:int' \
   --stages/stage1/callbacks_params/fast_run/num_epoch_steps='2:int' \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
   --logdir=${LOGDIR}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config0.yml \
@@ -839,7 +839,7 @@ rm -rf ./tests/logs/_tests_dl_callbacks ${EXP_OUTPUT}
 LOG_MSG='pipeline 23'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --stages/stage1/stage_params/num_epochs='2:int' \
   --stages/stage1/callbacks_params/optimizer/use_fast_zero_grad='1:bool' \

bin/tests/check_dl_core_overfit_callback.sh

@@ -63,7 +63,7 @@ function check_line_counts {
 LOG_MSG='pipeline 01'
 echo ${LOG_MSG}
 
-PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+PYTHONPATH=./examples:.:${PYTHONPATH} \
   python catalyst/dl/scripts/run.py \
   --expdir=${EXPDIR} \
   --config=${EXPDIR}/config30.yml \