diff --git a/RELEASE.md b/RELEASE.md index 0a8f921358..fc0281cc1c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -23,6 +23,7 @@ * Removed `.coveragerc` from the Kedro project template. `coverage` settings are now given in `pyproject.toml`. * Fixed a bug where packaging or pulling a modular pipeline with the same name as the project's package name would throw an error (or silently pass without including the pipeline source code in the wheel file). * Removed unintentional dependency on `git`. +* Fixed an issue where nested pipeline configuration was not included in the packaged pipeline. ## Minor breaking changes to the API diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py index 883167e953..e2b54529ce 100644 --- a/kedro/framework/cli/pipeline.py +++ b/kedro/framework/cli/pipeline.py @@ -667,7 +667,7 @@ def _package_pipeline( # pylint: disable=too-many-arguments # config files not to confuse users and avoid useless file copies configs_to_package = _find_config_files( artifacts_to_package.pipeline_conf, - [f"parameters*/**/{pipeline_name}.yml", f"parameters*/**/{pipeline_name}/*"], + [f"parameters*/**/{pipeline_name}.yml", f"parameters*/**/{pipeline_name}/**/*"], ) source_paths = ( @@ -901,8 +901,9 @@ def _generate_wheel_file( cls = exc.__class__ raise KedroCliError(f"{cls.__module__}.{cls.__qualname__}: {exc}") from exc + config_files = [str(file) for file in conf_target.rglob("*") if file.is_file()] setup_file = _generate_setup_file( - package_name, version, install_requires, temp_dir_path + package_name, version, install_requires, temp_dir_path, config_files ) package_file = destination / _get_wheel_name(name=package_name, version=version) @@ -925,18 +926,15 @@ def _generate_wheel_file( def _generate_setup_file( - package_name: str, version: str, install_requires: List[str], output_dir: Path + package_name: str, + version: str, + install_requires: List[str], + output_dir: Path, + config_files: List[str], ) -> Path: setup_file = output_dir / "setup.py" - package_data = { - package_name: [ - "README.md", - "config/parameters*", - "config/**/parameters*", - "config/parameters*/**", - "config/parameters*/**/*", - ] - } + + package_data = {package_name: ["README.md"] + config_files} setup_file_context = dict( name=package_name, version=version, diff --git a/tests/framework/cli/pipeline/test_pipeline_package.py b/tests/framework/cli/pipeline/test_pipeline_package.py index c53916f808..f1894de041 100644 --- a/tests/framework/cli/pipeline/test_pipeline_package.py +++ b/tests/framework/cli/pipeline/test_pipeline_package.py @@ -334,6 +334,66 @@ def test_package_modular_pipeline_with_nested_parameters( assert "retail/config/parameters/retail.yml" in wheel_contents assert "retail/config/parameters/retail_banking.yml" not in wheel_contents + def test_package_pipeline_with_deep_nested_parameters( + self, fake_repo_path, fake_project_cli, fake_metadata + ): + CliRunner().invoke( + fake_project_cli, ["pipeline", "create", "retail"], obj=fake_metadata + ) + deep_nested_param_path = Path( + fake_repo_path / "conf" / "base" / "parameters" / "deep" / "retail" + ) + deep_nested_param_path.mkdir(parents=True, exist_ok=True) + (deep_nested_param_path / "params1.yml").touch() + + deep_nested_param_path2 = Path( + fake_repo_path / "conf" / "base" / "parameters" / "retail" / "deep" + ) + deep_nested_param_path2.mkdir(parents=True, exist_ok=True) + (deep_nested_param_path2 / "params1.yml").touch() + + deep_nested_param_path3 = Path( + fake_repo_path / "conf" / "base" / "parameters" / "deep" + ) + deep_nested_param_path3.mkdir(parents=True, exist_ok=True) + (deep_nested_param_path3 / "retail.yml").touch() + + super_deep_nested_param_path = Path( + fake_repo_path + / "conf" + / "base" + / "parameters" + / "a" + / "b" + / "c" + / "d" + / "retail" + ) + super_deep_nested_param_path.mkdir(parents=True, exist_ok=True) + (super_deep_nested_param_path / "params3.yml").touch() + result = CliRunner().invoke( + fake_project_cli, ["pipeline", "package", "retail"], obj=fake_metadata + ) + + assert result.exit_code == 0 + assert "Pipeline `retail` packaged!" in result.output + + wheel_location = fake_repo_path / "src" / "dist" + assert f"Location: {wheel_location}" in result.output + + wheel_name = _get_wheel_name(name="retail", version="0.1") + wheel_file = wheel_location / wheel_name + assert wheel_file.is_file() + assert len(list(wheel_location.iterdir())) == 1 + + # pylint: disable=consider-using-with + wheel_contents = set(ZipFile(str(wheel_file)).namelist()) + assert "retail/config/parameters/deep/retail/params1.yml" in wheel_contents + assert "retail/config/parameters/retail/deep/params1.yml" in wheel_contents + assert "retail/config/parameters/retail.yml" in wheel_contents + assert "retail/config/parameters/deep/retail.yml" in wheel_contents + assert "retail/config/parameters/a/b/c/d/retail/params3.yml" in wheel_contents + def test_pipeline_package_version( self, fake_repo_path, fake_package_path, fake_project_cli, fake_metadata ):