Added: End of line detection and configuration. (#1407)

Co-authored-by: Philippe Ombredanne <pombredanne@nexb.com>
Co-authored-by: Francisco Molina <franciscojose.molina@alten.es>

.gitattributes

@@ -0,0 +1,7 @@
+*           text=auto
+# CRLF sets for test files! Or tests will fail on CI/CD
+*crlf.txt   text eol=crlf
+simple-with-newline-crlf.txt        text eol=crlf
+simple-with-newline.txt             text eol=lf
+*_crlf_newlines.txt                 text eol=crlf
+*_lf_newlines.txt                   text eol=lf

.gitignore

@@ -44,6 +44,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+.pytest_cache
 
 # Translations
 *.mo

.travis.yml

@@ -13,8 +13,10 @@ jobs:
         env: TOXENV=py37
       - python: 3.8
         env: TOXENV=py38
-      - python: pypy3
-        env: TOXENV=pypy3
+# Travis pypy3 excluded as outdated.
+# Currently done on github, will be resumed after Travis update.
+#      - python: pypy3
+#        env: TOXENV=pypy3
       - name: "Python 3.6 on Windows"
         os: windows
         language: shell

cookiecutter/generate.py

@@ -170,9 +170,20 @@ def generate_file(project_dir, infile, context, env, skip_if_file_exists=False):
             raise
         rendered_file = tmpl.render(**context)
 
+        # Detect original file newline to output the rendered file
+        # note: newline='' ensures newlines are not converted
+        with io.open(infile, 'r', encoding='utf-8', newline='') as rd:
+            rd.readline()  # Read the first line to load 'newlines' value
+
+            # Use `_new_lines` overwrite from context, if configured.
+            newline = rd.newlines
+            if context['cookiecutter'].get('_new_lines', False):
+                newline = context['cookiecutter']['_new_lines']
+                logger.debug('Overwriting end line character with %s', newline)
+
         logger.debug('Writing contents to file %s', outfile)
 
-        with io.open(outfile, 'w', encoding='utf-8') as fh:
+        with io.open(outfile, 'w', encoding='utf-8', newline=newline) as fh:
             fh.write(rendered_file)
 
     # Apply file permissions to output file

docs/advanced/index.rst

@@ -22,3 +22,4 @@ Various advanced topics regarding cookiecutter usage.
    dict_variables
    template_extensions
    directories
+   new_line_characters

docs/advanced/new_line_characters.rst

@@ -0,0 +1,25 @@
+.. _new-lines:
+
+Working with line-ends special symbols LF/CRLF
+----------------------------------------------
+
+*New in Cookiecutter 2.0*
+
+Before version 2.0 Cookiecutter silently used system line end character.
+LF for POSIX and CRLF for Windows. Since version 2.0 this behaviour changed
+and now can be forced at template level.
+
+By default Cookiecutter now check every file at render stage and use same line
+end as in source. This allow template developers to have both types of files in
+the same template. Developers should correctly configure their `.gitattributes`
+file to avoid line-end character overwrite by git.
+
+Special template variable `_new_lines` was added in Cookiecutter 2.0.
+Acceptable variables: `'\n\r'` for CRLF and `'\n'` for POSIX.
+
+Here is example how to force line endings to CRLF on any deployment::
+
+    {
+        "project_slug": "sample",
+        "_new_lines": "\n\r"
+    }

tests/files/{% if cookiecutter.generate_file == 'y' %}cheese.txt{% endif %}

@@ -0,0 +1 @@
+Testing that generate_file was {{ cookiecutter.generate_file }}

tests/files/{{cookiecutter.generate_file}}.txt

@@ -0,0 +1 @@
+Testing {{ cookiecutter.generate_file }}

tests/files/{{cookiecutter.generate_file}}_crlf_newlines.txt

@@ -0,0 +1,3 @@
+newline is CRLF
+newline is CRLF
+newline is CRLF

tests/files/{{cookiecutter.generate_file}}_lf_newlines.txt

@@ -0,0 +1,2 @@
+newline is LF
+newline is LF

tests/test-generate-files-line-end/cookiecutter.json

@@ -0,0 +1,9 @@
+{
+    "full_name": "Philippe Ombredanne",
+    "year": "2015",
+    "color": "blue",
+    "letter": "D",
+    "folder_name": "im_a.dir",
+    "filename": "im_a.file",
+    "test_name": "output_folder"
+}

tests/test-generate-files-line-end/{{cookiecutter.test_name}}/folder/in_folder.txt

@@ -0,0 +1 @@
+The color is {{ cookiecutter.color }} and the letter is {{ cookiecutter.letter }}.

tests/test-generate-files-line-end/{{cookiecutter.test_name}}/something.txt

@@ -0,0 +1,3 @@
+Hi!
+My name is {{ cookiecutter.full_name }}.
+It is {{ cookiecutter.year }}.

tests/test-generate-files-line-end/{{cookiecutter.test_name}}/{{cookiecutter.folder_name}}/{{cookiecutter.filename}}.py

@@ -0,0 +1,2 @@
+"""Sample file to be created through a cookiecutter run."""
+print("This is the contents of {{ cookiecutter.filename }}.py.")

tests/test-generate-files/input{{cookiecutter.food}}/simple-with-newline-crlf.txt

@@ -0,0 +1,3 @@
+newline is CRLF
+newline is CRLF
+newline is CRLF

tests/test-generate-files/input{{cookiecutter.food}}/simple-with-newline.txt

@@ -1 +1,2 @@
-I eat {{ cookiecutter.food }}
+newline is LF
+newline is LF

tests/test_generate_file.py

@@ -20,6 +20,10 @@ def tear_down():
     yield
     if os.path.exists('tests/files/cheese.txt'):
         os.remove('tests/files/cheese.txt')
+    if os.path.exists('tests/files/cheese_lf_newlines.txt'):
+        os.remove('tests/files/cheese_lf_newlines.txt')
+    if os.path.exists('tests/files/cheese_crlf_newlines.txt'):
+        os.remove('tests/files/cheese_crlf_newlines.txt')
 
 
 @pytest.fixture
@@ -32,9 +36,12 @@ def env():
 
 def test_generate_file(env):
     """Verify simple file is generated with rendered context data."""
-    infile = 'tests/files/{{generate_file}}.txt'
+    infile = 'tests/files/{{cookiecutter.generate_file}}.txt'
     generate.generate_file(
-        project_dir=".", infile=infile, context={'generate_file': 'cheese'}, env=env
+        project_dir=".",
+        infile=infile,
+        context={'cookiecutter': {'generate_file': 'cheese'}},
+        env=env,
     )
     assert os.path.isfile('tests/files/cheese.txt')
     with open('tests/files/cheese.txt', 'rt') as f:
@@ -74,9 +81,14 @@ def test_generate_file_with_true_condition(env):
 
     This test has positive answer, so file should be rendered.
     """
-    infile = 'tests/files/{% if generate_file == \'y\' %}cheese.txt{% endif %}'
+    infile = (
+        'tests/files/{% if cookiecutter.generate_file == \'y\' %}cheese.txt{% endif %}'
+    )
     generate.generate_file(
-        project_dir=".", infile=infile, context={'generate_file': 'y'}, env=env
+        project_dir=".",
+        infile=infile,
+        context={'cookiecutter': {'generate_file': 'y'}},
+        env=env,
     )
     assert os.path.isfile('tests/files/cheese.txt')
     with open('tests/files/cheese.txt', 'rt') as f:
@@ -89,9 +101,14 @@ def test_generate_file_with_false_condition(env):
 
     This test has negative answer, so file should not be rendered.
     """
-    infile = 'tests/files/{% if generate_file == \'y\' %}cheese.txt{% endif %}'
+    infile = (
+        'tests/files/{% if cookiecutter.generate_file == \'y\' %}cheese.txt{% endif %}'
+    )
     generate.generate_file(
-        project_dir=".", infile=infile, context={'generate_file': 'n'}, env=env
+        project_dir=".",
+        infile=infile,
+        context={'cookiecutter': {'generate_file': 'n'}},
+        env=env,
     )
     assert not os.path.isfile('tests/files/cheese.txt')
 
@@ -117,3 +134,39 @@ def test_generate_file_verbose_template_syntax_error(env, expected_msg):
             env=env,
         )
     assert str(exception.value) == expected_msg
+
+
+def test_generate_file_does_not_translate_lf_newlines_to_crlf(env, tmp_path):
+    """Verify that file generation use same line ending, as in source file."""
+    infile = 'tests/files/{{cookiecutter.generate_file}}_lf_newlines.txt'
+    generate.generate_file(
+        project_dir=".",
+        infile=infile,
+        context={'cookiecutter': {'generate_file': 'cheese'}},
+        env=env,
+    )
+
+    # this generated file should have a LF line ending
+    gf = 'tests/files/cheese_lf_newlines.txt'
+    with open(gf, 'r', encoding='utf-8', newline='') as f:
+        simple_text = f.readline()
+    assert simple_text == 'newline is LF\n'
+    assert f.newlines == '\n'
+
+
+def test_generate_file_does_not_translate_crlf_newlines_to_lf(env):
+    """Verify that file generation use same line ending, as in source file."""
+    infile = 'tests/files/{{cookiecutter.generate_file}}_crlf_newlines.txt'
+    generate.generate_file(
+        project_dir=".",
+        infile=infile,
+        context={'cookiecutter': {'generate_file': 'cheese'}},
+        env=env,
+    )
+
+    # this generated file should have a CRLF line ending
+    gf = 'tests/files/cheese_crlf_newlines.txt'
+    with open(gf, 'r', encoding='utf-8', newline='') as f:
+        simple_text = f.readline()
+    assert simple_text == 'newline is CRLF\r\n'
+    assert f.newlines == '\r\n'