Skip to content

Commit

Permalink
Removed wildcard dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
kamalsharma2 committed Jan 17, 2022
1 parent 460fee0 commit 9c109b6
Showing 1 changed file with 7 additions and 7 deletions.
14 changes: 7 additions & 7 deletions horovod/spark/common/store.py
Expand Up @@ -267,9 +267,6 @@ def get_checkpoint_filename(self):
def get_logs_subdir(self):
return 'logs'

def get_wildcard(self):
return '*'

def _get_full_path_or_default(self, path, default_key):
if path is not None:
return self.get_full_path(path)
Expand Down Expand Up @@ -315,13 +312,17 @@ def sync_fn(self, run_id):

def fn(local_run_path):
print(f"Syncing dir {local_run_path} to dir {run_path}")
if self.fs.exists(run_path):
local_run_path = os.path.join(local_run_path, self.get_wildcard())
self.copy(local_run_path, run_path, recursive=True, overwrite=True)

return fn

def copy(self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK,**kwargs):
"""
This method copies the contents of the local source directory to the target directory.
This is different from the fsspec's put() because it does not copy the source folder
to the target directory in the case when target directory already exists.
"""

from fsspec.implementations.local import LocalFileSystem, make_path_posix
from fsspec.utils import other_paths

Expand All @@ -334,9 +335,8 @@ def copy(self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK,**kwarg
lpath = make_path_posix(lpath)
fs = LocalFileSystem()
lpaths = fs.expand_path(lpath, recursive=recursive)
is_wildcard = True if lpath.endswith('/*') else False
rpaths = other_paths(
lpaths, rpath, exists=isinstance(rpath, str) and self.fs.isdir(rpath) and not is_wildcard
lpaths, rpath
)

callback.set_size(len(rpaths))
Expand Down

0 comments on commit 9c109b6

Please sign in to comment.