From de9b8b5d45eb6365b3d2c6c01352e9c3aa5f3d99 Mon Sep 17 00:00:00 2001 From: Spencer Pearson Date: Tue, 8 Nov 2022 11:27:21 -0700 Subject: [PATCH 1/3] test(artifacts): skip flaky test :( --- tests/unit_tests/test_wandb_artifacts_full.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit_tests/test_wandb_artifacts_full.py b/tests/unit_tests/test_wandb_artifacts_full.py index 2a4585f32a1..16c2a7ac570 100644 --- a/tests/unit_tests/test_wandb_artifacts_full.py +++ b/tests/unit_tests/test_wandb_artifacts_full.py @@ -222,6 +222,12 @@ def test_artifact_wait_failure(wandb_init, timeout): run.finish() +@pytest.mark.skip( + reason="TODO(spencerpearson): this test passes locally, but flakes in CI. After much investigation, I still have no clue.", + # examples of flakes: + # https://app.circleci.com/pipelines/github/wandb/wandb/16334/workflows/319d3e58-853e-46ec-8a3f-088cac41351c/jobs/325741/tests#failed-test-0 + # https://app.circleci.com/pipelines/github/wandb/wandb/16392/workflows/b26b3e63-c8d8-45f4-b7db-00f84b11f8b8/jobs/327312 +) def test_artifact_metadata_save(wandb_init, relay_server): # Test artifact metadata sucessfully saved for len(numpy) > 32 dummy_metadata = np.array([0] * 33) From c48551e3a74b84b8d0dc1f76b2a82f133353d903 Mon Sep 17 00:00:00 2001 From: Spencer Pearson Date: Thu, 10 Nov 2022 10:45:14 -0700 Subject: [PATCH 2/3] xfail, not skip --- tests/unit_tests/test_wandb_artifacts_full.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_wandb_artifacts_full.py b/tests/unit_tests/test_wandb_artifacts_full.py index 16c2a7ac570..37c06fcdb32 100644 --- a/tests/unit_tests/test_wandb_artifacts_full.py +++ b/tests/unit_tests/test_wandb_artifacts_full.py @@ -222,7 +222,7 @@ def test_artifact_wait_failure(wandb_init, timeout): run.finish() -@pytest.mark.skip( +@pytest.mark.xfail( reason="TODO(spencerpearson): this test passes locally, but flakes in CI. After much investigation, I still have no clue.", # examples of flakes: # https://app.circleci.com/pipelines/github/wandb/wandb/16334/workflows/319d3e58-853e-46ec-8a3f-088cac41351c/jobs/325741/tests#failed-test-0 From 35bc4ade9a6e9611a97b7226c1f88af8683247de Mon Sep 17 00:00:00 2001 From: Hugh Wimberly Date: Wed, 23 Nov 2022 01:29:11 -0800 Subject: [PATCH 3/3] Remove extra line --- tests/unit_tests/test_wandb_artifacts_full.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/test_wandb_artifacts_full.py b/tests/unit_tests/test_wandb_artifacts_full.py index 89d45bc4727..37c06fcdb32 100644 --- a/tests/unit_tests/test_wandb_artifacts_full.py +++ b/tests/unit_tests/test_wandb_artifacts_full.py @@ -222,7 +222,6 @@ def test_artifact_wait_failure(wandb_init, timeout): run.finish() - @pytest.mark.xfail( reason="TODO(spencerpearson): this test passes locally, but flakes in CI. After much investigation, I still have no clue.", # examples of flakes: