From 3b10993ced861db862320f684ae4fdf25350140b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Mon, 2 Nov 2020 13:46:40 +0100 Subject: [PATCH 1/7] feat: add from_dict/from_yaml utility functions --- setup.py | 2 +- test_fixtures/config.yml | 3 + tests/test_config.py | 41 +++++++++ torch_audiomentations/__init__.py | 1 + torch_audiomentations/utils/config.py | 124 ++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 test_fixtures/config.yml create mode 100644 tests/test_config.py create mode 100644 torch_audiomentations/utils/config.py diff --git a/setup.py b/setup.py index 13b3fbb7..35634142 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def find_version(*file_paths): exclude=["build", "scripts", "dist", "images", "test_fixtures", "tests"] ), install_requires=["torch>=1.2.0"], - tests_require=["pytest", "pytest-cov"], + tests_require=["pytest", "pytest-cov", "PyYAML"], python_requires=">=3.6", classifiers=[ "Programming Language :: Python :: 3", diff --git a/test_fixtures/config.yml b/test_fixtures/config.yml new file mode 100644 index 00000000..2e4057ce --- /dev/null +++ b/test_fixtures/config.yml @@ -0,0 +1,3 @@ +Gain: + min_gain_in_db: -12.0 + mode: per_channel \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..df1d3345 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,41 @@ +import unittest + +from tests.utils import TEST_FIXTURES_DIR +from torch_audiomentations import from_dict, from_yaml +from torch_audiomentations import Gain + + +# TODO: uncomment once Compose is available +# https://github.com/asteroid-team/torch-audiomentations/issues/23 +# from torch_audiomentations import Compose + + +class TestFromConfig(unittest.TestCase): + def test_from_dict(self): + config = {"Gain": {"min_gain_in_db": -12.0, "mode": "per_channel"}} + transform = from_dict(config) + + assert isinstance(transform, Gain) + assert transform.min_gain_in_db == -12.0 + assert transform.max_gain_in_db == 6.0 + assert transform.mode == "per_channel" + + # TODO: uncomment once Compose is available + # https://github.com/asteroid-team/torch-audiomentations/issues/23 + # def test_from_dict_compose(self): + # config = { + # "Gain": {"min_gain_in_db": -12.0, "mode": "per_channel"}, + # "PolarityInversion": {}, + # } + + # transform = from_dict(config) + # assert isinstance(transform, Compose) + + def test_from_yaml(self): + file_yml = TEST_FIXTURES_DIR / "config.yml" + transform = from_yaml(file_yml) + + assert isinstance(transform, Gain) + assert transform.min_gain_in_db == -12.0 + assert transform.max_gain_in_db == 6.0 + assert transform.mode == "per_channel" diff --git a/torch_audiomentations/__init__.py b/torch_audiomentations/__init__.py index 9cfd3cc4..7112e3af 100644 --- a/torch_audiomentations/__init__.py +++ b/torch_audiomentations/__init__.py @@ -3,5 +3,6 @@ from .augmentations.peak_normalization import PeakNormalization from .utils.convolution import convolve +from .utils.config import from_dict, from_yaml __version__ = "0.3.0" diff --git a/torch_audiomentations/utils/config.py b/torch_audiomentations/utils/config.py new file mode 100644 index 00000000..a4b9ee71 --- /dev/null +++ b/torch_audiomentations/utils/config.py @@ -0,0 +1,124 @@ +import warnings +from typing import Any, Dict, Mapping, Text, Union +from pathlib import Path +import torch_audiomentations +from torch_audiomentations.core.transforms_interface import BaseWaveformTransform + +# TODO: remove this try/except/else once Compose is available +# https://github.com/asteroid-team/torch-audiomentations/issues/23 +try: + from torch_audiomentations import Compose +except ImportError: + COMPOSE_NOT_IMPLEMENTED = True +else: + COMPOSE_NOT_IMPLEMENTED = False + +# TODO: define this elsewhere? +# TODO: update when a new type of transform is added (e.g. BaseSpectrogramTransform?) +# TODO: remove this if/else once Compose is available +# https://github.com/asteroid-team/torch-audiomentations/issues/23 +if COMPOSE_NOT_IMPLEMENTED: + Transform = Union[BaseWaveformTransform] +else: + Transform = Union[BaseWaveformTransform, Compose] + + +def from_dict(config: Dict[Text, Dict[Text, Any]]) -> Transform: + """Instantiate a transform from a configuration dictionary. + + `from_dict` can be used to instantiate a transform from its class name. + For instance, these two pieces of code are equivalent: + + >>> from torch_audiomentations import TransformClassName + >>> transform = TransformClassName(param_name=param_value, ...) + + >>> transform = from_dict({"TransformClassName": {"param_name": param_value, ...}}) + + Transforms composition is also supported: + + >>> compose = from_dict({"FirstTransform": {"param": value}, + ... "SecondTransform": {"param": value}}) + + :param config: configuration - a configuration dictionary + :returns: A transform. + :rtype Transform: + """ + + if len(config) > 1: + + # TODO: remove this once Compose is available + # https://github.com/asteroid-team/torch-audiomentations/issues/23 + if COMPOSE_NOT_IMPLEMENTED: + raise ValueError( + "torch_audiomentations does not implement Compose transforms" + ) + + # dictionary order is guaranteed to be insertion order since Python 3.7, + # and it was already the case in Python 3.6 but not officially. + # therefore, when `config` refers to more than one transform, we create + # a Compose transform using the dictionary order + + transforms = [ + from_dict({TransformClassName: transform_params}) + for TransformClassName, transform_params in config.items() + ] + return Compose(transforms) + + TransformClassName, transform_params = config.popitem() + + try: + TransformClass = getattr(torch_audiomentations, TransformClassName) + except AttributeError: + raise ValueError( + f"torch_audiomentations does not implement {TransformClassName} transform." + ) + + if not isinstance(transform_params, dict): + raise ValueError( + "Transform parameters must be provided as {'param_name': param_value'} dictionary." + ) + + return TransformClass(**transform_params) + + +def from_yaml(file_yml: Union[Path, Text]) -> Transform: + """Instantiate a transform from a YAML configuration file. + + `from_yaml` can be used to instantiate a transform from a YAML file. + For instance, these two pieces of code are equivalent: + + >>> from torch_audiomentations import TransformClassName + >>> transform = TransformClassName(param_name=param_value, ...) + + >>> transform = from_yaml("config.yml") + + where the content of `config.yml` is something like: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # config.yml + TransformClassName: + param_name: param_value + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Transforms composition is also supported: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # config.yml + FirstTransform: + param: value + SecondTransform: + param: value + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :param file_yml: configuration file - a path to a YAML file with the following structure: + :returns: A transform. + :rtype Transform: + """ + + try: + import yaml + except ImportError as e: + raise ImportError("") + + with open(file_yml, "r") as f: + config = yaml.load(f, Loader=yaml.SafeLoader) + + return from_dict(config) From bed6209a41af62d995e1ed6e52fc1c122e66e91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Mon, 2 Nov 2020 13:56:53 +0100 Subject: [PATCH 2/7] fix: clarify PyYAML requirement error. --- torch_audiomentations/utils/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torch_audiomentations/utils/config.py b/torch_audiomentations/utils/config.py index a4b9ee71..c767e041 100644 --- a/torch_audiomentations/utils/config.py +++ b/torch_audiomentations/utils/config.py @@ -116,7 +116,9 @@ def from_yaml(file_yml: Union[Path, Text]) -> Transform: try: import yaml except ImportError as e: - raise ImportError("") + raise ImportError( + "PyYAML package is needed by `from_yaml`: please install it first." + ) with open(file_yml, "r") as f: config = yaml.load(f, Loader=yaml.SafeLoader) From 0783056f5330e84ef2cf46ce42280dea42c8d492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Mon, 2 Nov 2020 13:59:16 +0100 Subject: [PATCH 3/7] fix: add PyYAML dependency --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 6946bbc7..98d0e7b0 100644 --- a/environment.yml +++ b/environment.yml @@ -24,3 +24,4 @@ dependencies: - setuptools>=41.0.0 - tqdm==4.49.0 - twine + - PyYAML==5.3.1 From d7c7e872610aeaff2c5bd74240b3b8145346a225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Tue, 3 Nov 2020 10:20:41 +0100 Subject: [PATCH 4/7] fix: fix CI --- .github/workflows/test_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_requirements.txt b/.github/workflows/test_requirements.txt index fb832230..77be14e0 100644 --- a/.github/workflows/test_requirements.txt +++ b/.github/workflows/test_requirements.txt @@ -9,3 +9,4 @@ py-cpuinfo>=7.0.0 pytest==5.3.4 pytest-cov==2.8.1 coverage==4.5.2 +PyYAML>=5.3.1 From fdb24823654fb829199b3c418317504dbcecf8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Wed, 4 Nov 2020 09:42:50 +0100 Subject: [PATCH 5/7] chore: streamline from_config API Bonus: this new API allows nested composition. --- test_fixtures/config.yml | 5 +- test_fixtures/config_compose.yml | 9 +++ tests/test_config.py | 40 +++++++--- torch_audiomentations/utils/config.py | 103 +++++++++++++------------- 4 files changed, 91 insertions(+), 66 deletions(-) create mode 100644 test_fixtures/config_compose.yml diff --git a/test_fixtures/config.yml b/test_fixtures/config.yml index 2e4057ce..94ad7b8c 100644 --- a/test_fixtures/config.yml +++ b/test_fixtures/config.yml @@ -1,3 +1,4 @@ -Gain: +transform: Gain +params: min_gain_in_db: -12.0 - mode: per_channel \ No newline at end of file + mode: per_channel diff --git a/test_fixtures/config_compose.yml b/test_fixtures/config_compose.yml new file mode 100644 index 00000000..1af641d6 --- /dev/null +++ b/test_fixtures/config_compose.yml @@ -0,0 +1,9 @@ +transform: Compose +params: + transforms: + - transform: Gain + params: + min_gain_in_db: -12.0 + mode: per_channel + - transform: PolarityInversion + shuffle: True diff --git a/tests/test_config.py b/tests/test_config.py index df1d3345..44c2e5e0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,7 +12,10 @@ class TestFromConfig(unittest.TestCase): def test_from_dict(self): - config = {"Gain": {"min_gain_in_db": -12.0, "mode": "per_channel"}} + config = { + "transform": "Gain", + "params": {"min_gain_in_db": -12.0, "mode": "per_channel"}, + } transform = from_dict(config) assert isinstance(transform, Gain) @@ -20,17 +23,6 @@ def test_from_dict(self): assert transform.max_gain_in_db == 6.0 assert transform.mode == "per_channel" - # TODO: uncomment once Compose is available - # https://github.com/asteroid-team/torch-audiomentations/issues/23 - # def test_from_dict_compose(self): - # config = { - # "Gain": {"min_gain_in_db": -12.0, "mode": "per_channel"}, - # "PolarityInversion": {}, - # } - - # transform = from_dict(config) - # assert isinstance(transform, Compose) - def test_from_yaml(self): file_yml = TEST_FIXTURES_DIR / "config.yml" transform = from_yaml(file_yml) @@ -39,3 +31,27 @@ def test_from_yaml(self): assert transform.min_gain_in_db == -12.0 assert transform.max_gain_in_db == 6.0 assert transform.mode == "per_channel" + + # TODO: uncomment once Compose is available + # https://github.com/asteroid-team/torch-audiomentations/issues/23 + + # def test_from_dict_compose(self): + # config = { + # "Compose": { + # "shuffle": True, + # "transforms": [ + # { + # "transform": "Gain", + # "params": {"min_gain_in_db": -12.0, "mode": "per_channel"}, + # }, + # {"transform": "PolarityInversion"}, + # ], + # } + # } + # transform = from_dict(config) + # assert isinstance(transform, Compose) + + # def test_from_yaml_compose(self): + # file_yml = TEST_FIXTURES_DIR / "config_compose.yml" + # transform = from_yaml(file_yml) + # assert isinstance(transform, Compose) diff --git a/torch_audiomentations/utils/config.py b/torch_audiomentations/utils/config.py index c767e041..0a5ea1ec 100644 --- a/torch_audiomentations/utils/config.py +++ b/torch_audiomentations/utils/config.py @@ -1,70 +1,50 @@ import warnings -from typing import Any, Dict, Mapping, Text, Union +from typing import Any, Dict, Text, Union from pathlib import Path import torch_audiomentations -from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -# TODO: remove this try/except/else once Compose is available -# https://github.com/asteroid-team/torch-audiomentations/issues/23 -try: - from torch_audiomentations import Compose -except ImportError: - COMPOSE_NOT_IMPLEMENTED = True -else: - COMPOSE_NOT_IMPLEMENTED = False +from torch_audiomentations.core.transforms_interface import BaseWaveformTransform # TODO: define this elsewhere? -# TODO: update when a new type of transform is added (e.g. BaseSpectrogramTransform?) -# TODO: remove this if/else once Compose is available +# TODO: update when a new type of transform is added (e.g. BaseSpectrogramTransform? Compose? OneOf? SomeOf?) # https://github.com/asteroid-team/torch-audiomentations/issues/23 -if COMPOSE_NOT_IMPLEMENTED: - Transform = Union[BaseWaveformTransform] -else: - Transform = Union[BaseWaveformTransform, Compose] +# https://github.com/asteroid-team/torch-audiomentations/issues/26 +Transform = Union[BaseWaveformTransform] -def from_dict(config: Dict[Text, Dict[Text, Any]]) -> Transform: +def from_dict(config: Dict[Text, Union[Text, Dict[Text, Any]]]) -> Transform: """Instantiate a transform from a configuration dictionary. `from_dict` can be used to instantiate a transform from its class name. For instance, these two pieces of code are equivalent: - >>> from torch_audiomentations import TransformClassName - >>> transform = TransformClassName(param_name=param_value, ...) + >>> from torch_audiomentations import Gain + >>> transform = Gain(min_gain_in_db=-12.0) - >>> transform = from_dict({"TransformClassName": {"param_name": param_value, ...}}) + >>> transform = from_dict({'transform': 'Gain', + ... 'params': {'min_gain_in_db': -12.0}}) Transforms composition is also supported: - >>> compose = from_dict({"FirstTransform": {"param": value}, - ... "SecondTransform": {"param": value}}) + >>> compose = from_dict( + ... {'transform': 'Compose', + ... 'params': {'transforms': [{'transform': 'Gain', + ... 'params': {'min_gain_in_db': -12.0, + ... 'mode': 'per_channel'}}, + ... {'transform': 'PolarityInversion'}], + ... 'shuffle': True}}) :param config: configuration - a configuration dictionary :returns: A transform. :rtype Transform: """ - if len(config) > 1: - - # TODO: remove this once Compose is available - # https://github.com/asteroid-team/torch-audiomentations/issues/23 - if COMPOSE_NOT_IMPLEMENTED: - raise ValueError( - "torch_audiomentations does not implement Compose transforms" - ) - - # dictionary order is guaranteed to be insertion order since Python 3.7, - # and it was already the case in Python 3.6 but not officially. - # therefore, when `config` refers to more than one transform, we create - # a Compose transform using the dictionary order - - transforms = [ - from_dict({TransformClassName: transform_params}) - for TransformClassName, transform_params in config.items() - ] - return Compose(transforms) - - TransformClassName, transform_params = config.popitem() + try: + TransformClassName: Text = config["transform"] + except KeyError: + raise ValueError( + "A (currently missing) 'transform' key should be used to define the transform type." + ) try: TransformClass = getattr(torch_audiomentations, TransformClassName) @@ -73,11 +53,23 @@ def from_dict(config: Dict[Text, Dict[Text, Any]]) -> Transform: f"torch_audiomentations does not implement {TransformClassName} transform." ) + transform_params: Dict = config.get("params", dict()) if not isinstance(transform_params, dict): raise ValueError( - "Transform parameters must be provided as {'param_name': param_value'} dictionary." + "Transform parameters must be provided as {'param_name': param_value} dictionary." ) + if TransformClassName in ["Compose", "OneOf", "SomeOf"]: + + # TODO: update once Compose, OneOf and SomeOf are available + # https://github.com/asteroid-team/torch-audiomentations/issues/23 + # https://github.com/asteroid-team/torch-audiomentations/issues/26 + # For now, we assume that API will expect a "transforms" key + transform_params["transforms"] = [ + from_dict(sub_transform_config) + for sub_transform_config in transform_params["transforms"] + ] + return TransformClass(**transform_params) @@ -87,25 +79,32 @@ def from_yaml(file_yml: Union[Path, Text]) -> Transform: `from_yaml` can be used to instantiate a transform from a YAML file. For instance, these two pieces of code are equivalent: - >>> from torch_audiomentations import TransformClassName - >>> transform = TransformClassName(param_name=param_value, ...) + >>> from torch_audiomentations import Gain + >>> transform = Gain(min_gain_in_db=-12.0, mode="per_channel") >>> transform = from_yaml("config.yml") where the content of `config.yml` is something like: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # config.yml - TransformClassName: - param_name: param_value + transform: Gain + params: + min_gain_in_db: -12.0 + mode: per_channel ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Transforms composition is also supported: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # config.yml - FirstTransform: - param: value - SecondTransform: - param: value + transform: Compose + params: + shuffle: True + transforms: + - transform: Gain + params: + min_gain_in_db: -12.0 + mode: per_channel + - transform: PolarityInversion ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :param file_yml: configuration file - a path to a YAML file with the following structure: From db1eea7f2d66e871a2a5e563e2bc2f71b8142836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Thu, 5 Nov 2020 14:42:01 +0100 Subject: [PATCH 6/7] feat: add support for Compose --- tests/test_config.py | 50 +++++++++++---------------- torch_audiomentations/utils/config.py | 11 ++---- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 44c2e5e0..1095e52f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,12 +2,7 @@ from tests.utils import TEST_FIXTURES_DIR from torch_audiomentations import from_dict, from_yaml -from torch_audiomentations import Gain - - -# TODO: uncomment once Compose is available -# https://github.com/asteroid-team/torch-audiomentations/issues/23 -# from torch_audiomentations import Compose +from torch_audiomentations import Gain, Compose class TestFromConfig(unittest.TestCase): @@ -32,26 +27,23 @@ def test_from_yaml(self): assert transform.max_gain_in_db == 6.0 assert transform.mode == "per_channel" - # TODO: uncomment once Compose is available - # https://github.com/asteroid-team/torch-audiomentations/issues/23 - - # def test_from_dict_compose(self): - # config = { - # "Compose": { - # "shuffle": True, - # "transforms": [ - # { - # "transform": "Gain", - # "params": {"min_gain_in_db": -12.0, "mode": "per_channel"}, - # }, - # {"transform": "PolarityInversion"}, - # ], - # } - # } - # transform = from_dict(config) - # assert isinstance(transform, Compose) - - # def test_from_yaml_compose(self): - # file_yml = TEST_FIXTURES_DIR / "config_compose.yml" - # transform = from_yaml(file_yml) - # assert isinstance(transform, Compose) + def test_from_dict_compose(self): + config = { + "Compose": { + "shuffle": True, + "transforms": [ + { + "transform": "Gain", + "params": {"min_gain_in_db": -12.0, "mode": "per_channel"}, + }, + {"transform": "PolarityInversion"}, + ], + } + } + transform = from_dict(config) + assert isinstance(transform, Compose) + + def test_from_yaml_compose(self): + file_yml = TEST_FIXTURES_DIR / "config_compose.yml" + transform = from_yaml(file_yml) + assert isinstance(transform, Compose) diff --git a/torch_audiomentations/utils/config.py b/torch_audiomentations/utils/config.py index 0a5ea1ec..cb6a9328 100644 --- a/torch_audiomentations/utils/config.py +++ b/torch_audiomentations/utils/config.py @@ -4,12 +4,12 @@ import torch_audiomentations from torch_audiomentations.core.transforms_interface import BaseWaveformTransform +from torch_audiomentations import Compose # TODO: define this elsewhere? -# TODO: update when a new type of transform is added (e.g. BaseSpectrogramTransform? Compose? OneOf? SomeOf?) -# https://github.com/asteroid-team/torch-audiomentations/issues/23 +# TODO: update when a new type of transform is added (e.g. BaseSpectrogramTransform? OneOf? SomeOf?) # https://github.com/asteroid-team/torch-audiomentations/issues/26 -Transform = Union[BaseWaveformTransform] +Transform = Union[BaseWaveformTransform, Compose] def from_dict(config: Dict[Text, Union[Text, Dict[Text, Any]]]) -> Transform: @@ -60,11 +60,6 @@ def from_dict(config: Dict[Text, Union[Text, Dict[Text, Any]]]) -> Transform: ) if TransformClassName in ["Compose", "OneOf", "SomeOf"]: - - # TODO: update once Compose, OneOf and SomeOf are available - # https://github.com/asteroid-team/torch-audiomentations/issues/23 - # https://github.com/asteroid-team/torch-audiomentations/issues/26 - # For now, we assume that API will expect a "transforms" key transform_params["transforms"] = [ from_dict(sub_transform_config) for sub_transform_config in transform_params["transforms"] From 1f31cc6e90fe0d1424efc1b21502b7e8ca4b97ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Bredin?= Date: Thu, 5 Nov 2020 14:46:58 +0100 Subject: [PATCH 7/7] fix: fix test_from_dict_compose (used old API) --- tests/test_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 1095e52f..43480e32 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -29,7 +29,8 @@ def test_from_yaml(self): def test_from_dict_compose(self): config = { - "Compose": { + "transform": "Compose", + "params": { "shuffle": True, "transforms": [ { @@ -38,7 +39,7 @@ def test_from_dict_compose(self): }, {"transform": "PolarityInversion"}, ], - } + }, } transform = from_dict(config) assert isinstance(transform, Compose)