Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added: End of line detection and configuration. #1407

Merged
merged 18 commits into from May 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 7 additions & 0 deletions .gitattributes
@@ -0,0 +1,7 @@
* text=auto
# CRLF sets for test files! Or tests will fail on CI/CD
*crlf.txt text eol=crlf
simple-with-newline-crlf.txt text eol=crlf
simple-with-newline.txt text eol=lf
*_crlf_newlines.txt text eol=crlf
*_lf_newlines.txt text eol=lf
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -44,6 +44,7 @@ nosetests.xml
coverage.xml
*,cover
.hypothesis/
.pytest_cache

# Translations
*.mo
Expand Down
6 changes: 4 additions & 2 deletions .travis.yml
Expand Up @@ -13,8 +13,10 @@ jobs:
env: TOXENV=py37
- python: 3.8
env: TOXENV=py38
- python: pypy3
env: TOXENV=pypy3
# Travis pypy3 excluded as outdated.
# Currently done on github, will be resumed after Travis update.
# - python: pypy3
# env: TOXENV=pypy3
- name: "Python 3.6 on Windows"
os: windows
language: shell
Expand Down
13 changes: 12 additions & 1 deletion cookiecutter/generate.py
Expand Up @@ -170,9 +170,20 @@ def generate_file(project_dir, infile, context, env, skip_if_file_exists=False):
raise
rendered_file = tmpl.render(**context)

# Detect original file newline to output the rendered file
# note: newline='' ensures newlines are not converted
with io.open(infile, 'r', encoding='utf-8', newline='') as rd:
rd.readline() # Read the first line to load 'newlines' value

# Use `_new_lines` overwrite from context, if configured.
newline = rd.newlines
if context['cookiecutter'].get('_new_lines', False):
newline = context['cookiecutter']['_new_lines']
logger.debug('Overwriting end line character with %s', newline)

logger.debug('Writing contents to file %s', outfile)

with io.open(outfile, 'w', encoding='utf-8') as fh:
with io.open(outfile, 'w', encoding='utf-8', newline=newline) as fh:
insspb marked this conversation as resolved.
Show resolved Hide resolved
fh.write(rendered_file)

# Apply file permissions to output file
Expand Down
1 change: 1 addition & 0 deletions docs/advanced/index.rst
Expand Up @@ -22,3 +22,4 @@ Various advanced topics regarding cookiecutter usage.
dict_variables
template_extensions
directories
new_line_characters
25 changes: 25 additions & 0 deletions docs/advanced/new_line_characters.rst
@@ -0,0 +1,25 @@
.. _new-lines:

Working with line-ends special symbols LF/CRLF
----------------------------------------------

*New in Cookiecutter 2.0*

Before version 2.0 Cookiecutter silently used system line end character.
LF for POSIX and CRLF for Windows. Since version 2.0 this behaviour changed
and now can be forced at template level.

By default Cookiecutter now check every file at render stage and use same line
end as in source. This allow template developers to have both types of files in
the same template. Developers should correctly configure their `.gitattributes`
file to avoid line-end character overwrite by git.

Special template variable `_new_lines` was added in Cookiecutter 2.0.
Acceptable variables: `'\n\r'` for CRLF and `'\n'` for POSIX.

Here is example how to force line endings to CRLF on any deployment::

{
"project_slug": "sample",
"_new_lines": "\n\r"
}
@@ -0,0 +1 @@
Testing that generate_file was {{ cookiecutter.generate_file }}

This file was deleted.

1 change: 1 addition & 0 deletions tests/files/{{cookiecutter.generate_file}}.txt
@@ -0,0 +1 @@
Testing {{ cookiecutter.generate_file }}
3 changes: 3 additions & 0 deletions tests/files/{{cookiecutter.generate_file}}_crlf_newlines.txt
@@ -0,0 +1,3 @@
newline is CRLF
newline is CRLF
newline is CRLF
2 changes: 2 additions & 0 deletions tests/files/{{cookiecutter.generate_file}}_lf_newlines.txt
@@ -0,0 +1,2 @@
newline is LF
newline is LF
1 change: 0 additions & 1 deletion tests/files/{{generate_file}}.txt

This file was deleted.

9 changes: 9 additions & 0 deletions tests/test-generate-files-line-end/cookiecutter.json
@@ -0,0 +1,9 @@
{
"full_name": "Philippe Ombredanne",
"year": "2015",
"color": "blue",
"letter": "D",
"folder_name": "im_a.dir",
"filename": "im_a.file",
"test_name": "output_folder"
}
@@ -0,0 +1 @@
The color is {{ cookiecutter.color }} and the letter is {{ cookiecutter.letter }}.
@@ -0,0 +1,3 @@
Hi!
My name is {{ cookiecutter.full_name }}.
It is {{ cookiecutter.year }}.
@@ -0,0 +1,2 @@
"""Sample file to be created through a cookiecutter run."""
print("This is the contents of {{ cookiecutter.filename }}.py.")
@@ -0,0 +1,3 @@
newline is CRLF
newline is CRLF
newline is CRLF
@@ -1 +1,2 @@
I eat {{ cookiecutter.food }}
newline is LF
newline is LF
65 changes: 59 additions & 6 deletions tests/test_generate_file.py
Expand Up @@ -20,6 +20,10 @@ def tear_down():
yield
if os.path.exists('tests/files/cheese.txt'):
os.remove('tests/files/cheese.txt')
if os.path.exists('tests/files/cheese_lf_newlines.txt'):
os.remove('tests/files/cheese_lf_newlines.txt')
if os.path.exists('tests/files/cheese_crlf_newlines.txt'):
os.remove('tests/files/cheese_crlf_newlines.txt')


@pytest.fixture
Expand All @@ -32,9 +36,12 @@ def env():

def test_generate_file(env):
"""Verify simple file is generated with rendered context data."""
infile = 'tests/files/{{generate_file}}.txt'
infile = 'tests/files/{{cookiecutter.generate_file}}.txt'
generate.generate_file(
project_dir=".", infile=infile, context={'generate_file': 'cheese'}, env=env
project_dir=".",
infile=infile,
context={'cookiecutter': {'generate_file': 'cheese'}},
env=env,
)
assert os.path.isfile('tests/files/cheese.txt')
with open('tests/files/cheese.txt', 'rt') as f:
Expand Down Expand Up @@ -74,9 +81,14 @@ def test_generate_file_with_true_condition(env):

This test has positive answer, so file should be rendered.
"""
infile = 'tests/files/{% if generate_file == \'y\' %}cheese.txt{% endif %}'
infile = (
'tests/files/{% if cookiecutter.generate_file == \'y\' %}cheese.txt{% endif %}'
)
generate.generate_file(
project_dir=".", infile=infile, context={'generate_file': 'y'}, env=env
project_dir=".",
infile=infile,
context={'cookiecutter': {'generate_file': 'y'}},
env=env,
)
assert os.path.isfile('tests/files/cheese.txt')
with open('tests/files/cheese.txt', 'rt') as f:
Expand All @@ -89,9 +101,14 @@ def test_generate_file_with_false_condition(env):

This test has negative answer, so file should not be rendered.
"""
infile = 'tests/files/{% if generate_file == \'y\' %}cheese.txt{% endif %}'
infile = (
'tests/files/{% if cookiecutter.generate_file == \'y\' %}cheese.txt{% endif %}'
)
generate.generate_file(
project_dir=".", infile=infile, context={'generate_file': 'n'}, env=env
project_dir=".",
infile=infile,
context={'cookiecutter': {'generate_file': 'n'}},
env=env,
)
assert not os.path.isfile('tests/files/cheese.txt')

Expand All @@ -117,3 +134,39 @@ def test_generate_file_verbose_template_syntax_error(env, expected_msg):
env=env,
)
assert str(exception.value) == expected_msg


def test_generate_file_does_not_translate_lf_newlines_to_crlf(env, tmp_path):
"""Verify that file generation use same line ending, as in source file."""
infile = 'tests/files/{{cookiecutter.generate_file}}_lf_newlines.txt'
generate.generate_file(
project_dir=".",
infile=infile,
context={'cookiecutter': {'generate_file': 'cheese'}},
env=env,
)

# this generated file should have a LF line ending
gf = 'tests/files/cheese_lf_newlines.txt'
with open(gf, 'r', encoding='utf-8', newline='') as f:
simple_text = f.readline()
assert simple_text == 'newline is LF\n'
assert f.newlines == '\n'


def test_generate_file_does_not_translate_crlf_newlines_to_lf(env):
"""Verify that file generation use same line ending, as in source file."""
infile = 'tests/files/{{cookiecutter.generate_file}}_crlf_newlines.txt'
generate.generate_file(
project_dir=".",
infile=infile,
context={'cookiecutter': {'generate_file': 'cheese'}},
env=env,
)

# this generated file should have a CRLF line ending
gf = 'tests/files/cheese_crlf_newlines.txt'
with open(gf, 'r', encoding='utf-8', newline='') as f:
simple_text = f.readline()
assert simple_text == 'newline is CRLF\r\n'
assert f.newlines == '\r\n'