fix(cli): Deflake wandb verify (#4438)

fix(cli): Deflake wandb verify

wandb/sdk/verify/verify.py

@@ -23,6 +23,11 @@
 MIN_RETRYS = 3
 CHECKMARK = "\u2705"
 RED_X = "\u274C"
+ID_PREFIX = wandb.util.generate_id()
+
+
+def nice_id(name):
+    return ID_PREFIX + "-" + name
 
 
 def print_results(
@@ -125,7 +130,9 @@ def check_run(api: Api) -> bool:
     f.write("test")
     f.close()
 
-    with wandb.init(reinit=True, config=config, project=PROJECT_NAME) as run:
+    with wandb.init(
+        id=nice_id("check_run"), reinit=True, config=config, project=PROJECT_NAME
+    ) as run:
         run_id = run.id
         entity = run.entity
         logged = True
@@ -180,7 +187,9 @@ def check_run(api: Api) -> bool:
     # TODO: (kdg) refactor this so it doesn't rely on an exception handler
     try:
         read_file = retry_fn(partial(prev_run.file, filepath))
-        read_file = read_file.download(replace=True)
+        # There's a race where the file hasn't been processed in the queue,
+        # we just retry until we get a download
+        read_file = retry_fn(partial(read_file.download, replace=True))
     except Exception:
         failed_test_strings.append(
             "Unable to download file. Check SQS configuration, topic configuration and bucket permissions."
@@ -261,7 +270,10 @@ def log_use_download_artifact(
     add_extra_file: bool,
 ) -> Tuple[bool, Optional["ArtifactAPI"], List[str]]:
     with wandb.init(
-        reinit=True, project=PROJECT_NAME, config={"test": "artifact log"}
+        id=nice_id("log_artifact"),
+        reinit=True,
+        project=PROJECT_NAME,
+        config={"test": "artifact log"},
     ) as log_art_run:
 
         if add_extra_file:
@@ -277,6 +289,7 @@ def log_use_download_artifact(
             return False, None, failed_test_strings
 
     with wandb.init(
+        id=nice_id("use_artifact"),
         project=PROJECT_NAME,
         config={"test": "artifact use"},
     ) as use_art_run:
@@ -363,7 +376,10 @@ def check_graphql_put(api: Api, host: str) -> Tuple[bool, Optional[str]]:
     f.write("test2")
     f.close()
     with wandb.init(
-        reinit=True, project=PROJECT_NAME, config={"test": "put to graphql"}
+        id=nice_id("graphql_put"),
+        reinit=True,
+        project=PROJECT_NAME,
+        config={"test": "put to graphql"},
     ) as run:
         wandb.save(gql_fp)
     public_api = wandb.Api()
@@ -378,7 +394,7 @@ def check_graphql_put(api: Api, host: str) -> Tuple[bool, Optional[str]]:
     try:
         read_file = retry_fn(partial(prev_run.file, gql_fp))
         url = read_file.url
-        read_file = read_file.download(replace=True)
+        read_file = retry_fn(partial(read_file.download, replace=True))
     except Exception:
         failed_test_strings.append(
             "Unable to read file successfully saved through a put request. Check SQS configurations, bucket permissions and topic configs."