From 3e869cd8639423367a8bb67a51af06018a482e8c Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Thu, 18 Mar 2021 15:28:20 -0700 Subject: [PATCH 1/5] Added CR LF to LF Conversion for gradlew --- samcli/lib/init/__init__.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py index 56a948c6b2..c71d387e43 100644 --- a/samcli/lib/init/__init__.py +++ b/samcli/lib/init/__init__.py @@ -3,6 +3,8 @@ """ import itertools import logging +import platform +import os from pathlib import Path @@ -90,6 +92,9 @@ def generate_project( try: LOG.debug("Baking a new template with cookiecutter with all parameters") cookiecutter(**params) + # Fix gradlew line ending issue caused by Windows git + # Putting it after cookiecutter as it will change the line ending + _fix_gradlew_line_ending(output_dir) except RepositoryNotFound as e: # cookiecutter.json is not found in the template. Let's just clone it directly without using cookiecutter # and call it done. @@ -104,3 +109,22 @@ def generate_project( raise InvalidLocationError(template=params["template"]) from e except CookiecutterException as e: raise GenerateProjectFailedError(project=name, provider_error=e) from e + + +def _fix_gradlew_line_ending(path): + if platform.system().lower() != "windows": + return + for subdirectory, _, files in os.walk(path): + for file in files: + if file != "gradlew": + continue + file_path = os.path.join(subdirectory, file) + _replace_to_unix_line_ending(file_path) + + +def _replace_to_unix_line_ending(file_path): + with open(file_path, "rb") as file: + content = file.read() + content = content.replace(b"\r\n", b"\n") + with open(file_path, "wb") as file: + file.write(content) From f5401766da70ba0c4e44dff562f0cbfe38d0314a Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Mon, 22 Mar 2021 11:07:28 -0700 Subject: [PATCH 2/5] Added convert_files_to_unix_line_endings to osutils --- samcli/lib/init/__init__.py | 23 +++-------------------- samcli/lib/utils/osutils.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py index c71d387e43..ed10dc6024 100644 --- a/samcli/lib/init/__init__.py +++ b/samcli/lib/init/__init__.py @@ -13,6 +13,7 @@ from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING from samcli.lib.utils.packagetype import ZIP +from samcli.lib.utils import osutils from .exceptions import GenerateProjectFailedError, InvalidLocationError from .arbitrary_project import generate_non_cookiecutter_project @@ -94,7 +95,8 @@ def generate_project( cookiecutter(**params) # Fix gradlew line ending issue caused by Windows git # Putting it after cookiecutter as it will change the line ending - _fix_gradlew_line_ending(output_dir) + if platform.system().lower() == "windows": + osutils.convert_files_to_unix_line_endings(output_dir, ["gradlew"]) except RepositoryNotFound as e: # cookiecutter.json is not found in the template. Let's just clone it directly without using cookiecutter # and call it done. @@ -109,22 +111,3 @@ def generate_project( raise InvalidLocationError(template=params["template"]) from e except CookiecutterException as e: raise GenerateProjectFailedError(project=name, provider_error=e) from e - - -def _fix_gradlew_line_ending(path): - if platform.system().lower() != "windows": - return - for subdirectory, _, files in os.walk(path): - for file in files: - if file != "gradlew": - continue - file_path = os.path.join(subdirectory, file) - _replace_to_unix_line_ending(file_path) - - -def _replace_to_unix_line_ending(file_path): - with open(file_path, "rb") as file: - content = file.read() - content = content.replace(b"\r\n", b"\n") - with open(file_path, "wb") as file: - file.write(content) diff --git a/samcli/lib/utils/osutils.py b/samcli/lib/utils/osutils.py index bdff5ea3b2..68b6fa02d1 100644 --- a/samcli/lib/utils/osutils.py +++ b/samcli/lib/utils/osutils.py @@ -8,6 +8,7 @@ import sys import tempfile from contextlib import contextmanager +from typing import List, Optional LOG = logging.getLogger(__name__) @@ -154,3 +155,21 @@ def copytree(source, destination, ignore=None): copytree(new_source, new_destination, ignore=ignore) else: shutil.copy2(new_source, new_destination) + + +def convert_files_to_unix_line_endings(path: str, target_files: Optional[List[str]] = None) -> None: + for subdirectory, _, files in os.walk(path): + for file in files: + if target_files is not None and file not in target_files: + continue + + file_path = os.path.join(subdirectory, file) + convert_to_unix_line_ending(file_path) + + +def convert_to_unix_line_ending(file_path: str) -> None: + with open(file_path, "rb") as file: + content = file.read() + content = content.replace(b"\r\n", b"\n") + with open(file_path, "wb") as file: + file.write(content) From 014e98e2fb6bb2d2a7889f6cf2c2eef660d10f2f Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Mon, 22 Mar 2021 11:09:04 -0700 Subject: [PATCH 3/5] Removed Unused Import --- samcli/lib/init/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py index ed10dc6024..1605e82040 100644 --- a/samcli/lib/init/__init__.py +++ b/samcli/lib/init/__init__.py @@ -4,7 +4,6 @@ import itertools import logging import platform -import os from pathlib import Path From 47725a82977ba80d2482f3eb2746bd7373e4e298 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Mon, 22 Mar 2021 11:18:29 -0700 Subject: [PATCH 4/5] Added More Comments --- samcli/lib/init/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py index 1605e82040..089d81f18e 100644 --- a/samcli/lib/init/__init__.py +++ b/samcli/lib/init/__init__.py @@ -92,8 +92,10 @@ def generate_project( try: LOG.debug("Baking a new template with cookiecutter with all parameters") cookiecutter(**params) - # Fix gradlew line ending issue caused by Windows git - # Putting it after cookiecutter as it will change the line ending + # Fixes gradlew line ending issue caused by Windows git + # gradlew is a shell script which should not have CR LF line endings + # Putting the conversion after cookiecutter as cookiecutter processing will also change the line endings + # https://github.com/cookiecutter/cookiecutter/pull/1407 if platform.system().lower() == "windows": osutils.convert_files_to_unix_line_endings(output_dir, ["gradlew"]) except RepositoryNotFound as e: From 6364cb560dda1dc539d48234a3d0f30d5c850317 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Wed, 24 Mar 2021 10:22:28 -0700 Subject: [PATCH 5/5] Updated Tests --- tests/integration/buildcmd/test_build_cmd.py | 13 +------------ tests/unit/lib/utils/test_osutils.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index b8e53dbe59..be22df5595 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -335,8 +335,6 @@ class TestBuildCommand_Java(BuildIntegBase): USING_GRADLEW_PATH = os.path.join("Java", "gradlew") USING_GRADLE_KOTLIN_PATH = os.path.join("Java", "gradle-kotlin") USING_MAVEN_PATH = os.path.join("Java", "maven") - WINDOWS_LINE_ENDING = b"\r\n" - UNIX_LINE_ENDING = b"\n" @parameterized.expand( [ @@ -401,7 +399,7 @@ def _test_with_building_java(self, runtime, code_path, expected_files, use_conta cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) cmdlist += ["--skip-pull-image"] if code_path == self.USING_GRADLEW_PATH and use_container and IS_WINDOWS: - self._change_to_unix_line_ending(os.path.join(self.test_data_path, self.USING_GRADLEW_PATH, "gradlew")) + osutils.convert_to_unix_line_ending(os.path.join(self.test_data_path, self.USING_GRADLEW_PATH, "gradlew")) LOG.info("Running Command: {}".format(cmdlist)) run_command(cmdlist, cwd=self.working_dir) @@ -464,15 +462,6 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files, lib_dir_contents = set(os.listdir(str(resource_artifact_dir.joinpath("lib")))) self.assertEqual(lib_dir_contents, expected_modules) - def _change_to_unix_line_ending(self, path): - with open(os.path.abspath(path), "rb") as open_file: - content = open_file.read() - - content = content.replace(self.WINDOWS_LINE_ENDING, self.UNIX_LINE_ENDING) - - with open(os.path.abspath(path), "wb") as open_file: - open_file.write(content) - @skipIf( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), diff --git a/tests/unit/lib/utils/test_osutils.py b/tests/unit/lib/utils/test_osutils.py index d7633509b0..fbcdb10a81 100644 --- a/tests/unit/lib/utils/test_osutils.py +++ b/tests/unit/lib/utils/test_osutils.py @@ -61,3 +61,19 @@ def test_must_return_sys_stdout(self): expected_stdout = sys.stdout.buffer self.assertEqual(expected_stdout, osutils.stdout()) + + +class Test_convert_files_to_unix_line_endings: + @patch("os.walk") + @patch("builtins.open") + def test_must_return_sys_stdout(self, patched_open, os_walk): + target_file = "target_file" + os_walk.return_value = [ + ("a", "_", ("file_a_1", "file_a_2", target_file)), + ("b", "_", ("file_b_1", target_file)), + ] + osutils.convert_files_to_unix_line_endings("path", [target_file]) + patched_open.assert_any_call(os.path.join("a", target_file), "rb") + patched_open.assert_any_call(os.path.join("b", target_file), "rb") + patched_open.assert_any_call(os.path.join("a", target_file), "wb") + patched_open.assert_any_call(os.path.join("b", target_file), "wb")