Skip to content

Commit

Permalink
Split test
Browse files Browse the repository at this point in the history
  • Loading branch information
@LysandreJik
LysandreJik committed Apr 12, 2022
1 parent 5f48d6c commit 6d27a15
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 68 deletions.
20 changes: 8 additions & 12 deletions src/huggingface_hub/hf_api.py
View file
Expand Up @@ -569,23 +569,19 @@ def _validate_or_retrieve_token(
function_name: Optional[str] = None,
):
"""
Args:
Retrieves and validates stored token or validates passed token.
Args:
token (``str``, `optional`):
Hugging Face token. Will default to the locally saved token if
not provided.
Hugging Face token. Will default to the locally saved token if not provided.
name (``str``, `optional`):
Name of the repository. This is deprecated in favor of repo_id
and will be removed in v0.7.
Name of the repository. This is deprecated in favor of repo_id and will be removed in v0.7.
function_name (``str``, `optional`):
If _validate_or_retrieve_token is called from a function, name
of that function to be passed inside deprecation warning.
If _validate_or_retrieve_token is called from a function, name of that function to be passed inside deprecation warning.
Returns:
Validated token and the name of the repository.
Raises:
:class:`EnvironmentError`: If the token is not passed and there's no
token saved locally. :class:`ValueError`: If organization token or
invalid token is passed.
:class:`EnvironmentError`: If the token is not passed and there's no token saved locally.
:class:`ValueError`: If organization token or invalid token is passed.
"""
if token is None or token is True:
token = HfFolder.get_token()
Expand Down Expand Up @@ -1851,8 +1847,8 @@ def get_token(cls) -> Optional[str]:
"""
Get token or None if not existent.
Note that a token can be also provided using the
`HUGGING_FACE_HUB_TOKEN` environment variable.
Note that a token can be also provided using the `HUGGING_FACE_HUB_TOKEN`
environment variable.
Returns:
`str` or `None`: The token, `None` if it doesn't exist.
Expand Down
152 changes: 96 additions & 56 deletions tests/test_repository.py
View file
Expand Up @@ -1075,18 +1075,12 @@ def test_is_tracked_with_lfs_with_pattern(self):
# This content is 20MB (over 10MB)
large_file = [100] * int(4e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
f.write(json.dumps(large_file))

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(small_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

os.makedirs(f"{WORKING_REPO_DIR}/dir", exist_ok=True)

with open(f"{WORKING_REPO_DIR}/dir/large_file.txt", "w+") as f:
Expand All @@ -1095,30 +1089,20 @@ def test_is_tracked_with_lfs_with_pattern(self):
with open(f"{WORKING_REPO_DIR}/dir/small_file.txt", "w+") as f:
f.write(json.dumps(small_file))

with open(f"{WORKING_REPO_DIR}/dir/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.auto_track_large_files("dir")
repo.auto_track_binary_files("dir")

self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
)
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
)
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
)
self.assertTrue(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "dir/large_file.txt"))
)
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "dir/small_file.txt"))
)
self.assertTrue(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "dir/binary_file.txt"))
)

def test_auto_track_large_files(self):
repo = Repository(WORKING_REPO_DIR)
Expand All @@ -1129,27 +1113,41 @@ def test_auto_track_large_files(self):
# This content is 20MB (over 10MB)
large_file = [100] * int(4e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
f.write(json.dumps(large_file))

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(small_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.auto_track_large_files()
repo.auto_track_binary_files()

self.assertTrue(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
)
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
)

def test_auto_track_binary_files(self):
repo = Repository(WORKING_REPO_DIR)

# This content is non-binary
non_binary_file = [100] * int(1e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/non_binary_file.txt", "w+") as f:
f.write(json.dumps(non_binary_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.auto_track_binary_files()

self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary)file.txt"))
)
self.assertTrue(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
)
Expand All @@ -1160,17 +1158,14 @@ def test_auto_track_large_files_ignored_with_gitignore(self):
# This content is 20MB (over 10MB)
large_file = [100] * int(4e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

# Test nested gitignores
os.makedirs(f"{WORKING_REPO_DIR}/directory")

with open(f"{WORKING_REPO_DIR}/.gitignore", "w+") as f:
f.write("large_file.txt\nbinary_file.txt")
f.write("large_file.txt")

with open(f"{WORKING_REPO_DIR}/directory/.gitignore", "w+") as f:
f.write("large_file_3.txt\nbinary_file_3.txt")
f.write("large_file_3.txt")

with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
f.write(json.dumps(large_file))
Expand All @@ -1186,20 +1181,6 @@ def test_auto_track_large_files_ignored_with_gitignore(self):

repo.auto_track_large_files()

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/binary_file_2.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/directory/binary_file_3.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/directory/binary_file_4.txt", "w+") as f:
f.write(binary_file)

repo.auto_track_binary_files()

# Large files
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
Expand All @@ -1219,6 +1200,35 @@ def test_auto_track_large_files_ignored_with_gitignore(self):
)
)

def test_auto_track_binary_files_ignored_with_gitignore(self):
repo = Repository(WORKING_REPO_DIR)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

# Test nested gitignores
os.makedirs(f"{WORKING_REPO_DIR}/directory")

with open(f"{WORKING_REPO_DIR}/.gitignore", "w+") as f:
f.write("binary_file.txt")

with open(f"{WORKING_REPO_DIR}/directory/.gitignore", "w+") as f:
f.write("binary_file_3.txt")

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/binary_file_2.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/directory/binary_file_3.txt", "w+") as f:
f.write(binary_file)

with open(f"{WORKING_REPO_DIR}/directory/binary_file_4.txt", "w+") as f:
f.write(binary_file)

repo.auto_track_binary_files()

# Binary files
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
Expand All @@ -1238,7 +1248,7 @@ def test_auto_track_large_files_ignored_with_gitignore(self):
)
)

def test_auto_track_files_through_git_add(self):
def test_auto_track_large_files_through_git_add(self):
repo = Repository(WORKING_REPO_DIR)

# This content is 5MB (under 10MB)
Expand All @@ -1247,18 +1257,12 @@ def test_auto_track_files_through_git_add(self):
# This content is 20MB (over 10MB)
large_file = [100] * int(4e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
f.write(json.dumps(large_file))

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(small_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.git_add(auto_lfs_track=True)

self.assertTrue(
Expand All @@ -1267,11 +1271,32 @@ def test_auto_track_files_through_git_add(self):
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
)

def test_auto_track_binary_files_through_git_add(self):
repo = Repository(WORKING_REPO_DIR)

# This content is non binary
non_binary_file = [100] * int(1e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(non_binary_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.git_add(auto_lfs_track=True)

self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary_file.txt"))
)
self.assertTrue(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
)

def test_auto_no_track_files_through_git_add(self):
def test_auto_no_track_large_files_through_git_add(self):
repo = Repository(WORKING_REPO_DIR)

# This content is 5MB (under 10MB)
Expand All @@ -1280,18 +1305,12 @@ def test_auto_no_track_files_through_git_add(self):
# This content is 20MB (over 10MB)
large_file = [100] * int(4e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
f.write(json.dumps(large_file))

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(small_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.git_add(auto_lfs_track=False)

self.assertFalse(
Expand All @@ -1300,6 +1319,27 @@ def test_auto_no_track_files_through_git_add(self):
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
)

def test_auto_no_track_binary_files_through_git_add(self):
repo = Repository(WORKING_REPO_DIR)

# This content is non-binary
non_binary_file = [100] * int(1e6)

# This content is binary (contains the null character)
binary_file = "\x00\x00\x00\x00"

with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
f.write(json.dumps(non_binary_file))

with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
f.write(binary_file)

repo.git_add(auto_lfs_track=False)

self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary_file.txt"))
)
self.assertFalse(
is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
)
Expand Down

0 comments on commit 6d27a15

Please sign in to comment.