Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(launch): improved git fetch time by specifying a refspec and depth=1 #4459

Merged
merged 3 commits into from Nov 8, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 20 additions & 1 deletion wandb/sdk/launch/utils.py
Expand Up @@ -414,6 +414,24 @@ def apply_patch(patch_string: str, dst_dir: str) -> None:
raise wandb.Error("Failed to apply diff.patch associated with run.")


def _make_refspec_from_version(version: Optional[str]) -> List[str]:
"""
Helper to create a refspec that checks for the existence of origin/main
and the version, if provided.
"""
refspec = [
"+refs/heads/main*:refs/remotes/origin/main*",
"+refs/heads/master*:refs/remotes/origin/master*",
]
if not version or version in ["main", "master"]:
return refspec
elif len(version) == 40: # if hash, only return hash
gtarpenning marked this conversation as resolved.
Show resolved Hide resolved
# TODO(gst): need better check for SHA here
return [f"+{version}"]
else:
return refspec + [f"+refs/heads/{version}*:refs/remotes/origin/{version}*"]


def _fetch_git_repo(dst_dir: str, uri: str, version: Optional[str]) -> str:
"""Clones the git repo at ``uri`` into ``dst_dir``.

Expand All @@ -428,7 +446,8 @@ def _fetch_git_repo(dst_dir: str, uri: str, version: Optional[str]) -> str:
_logger.info("Fetching git repo")
repo = git.Repo.init(dst_dir)
origin = repo.create_remote("origin", uri)
origin.fetch()
refspec = _make_refspec_from_version(version)
origin.fetch(refspec=refspec, depth=1)

if version is not None:
try:
Expand Down