Run tests several times in CI

I've been wanting to do this for ages to catch/fix flaky tests sooner, but the tests were always too flaky and/or too slow (especially with the old vmactions), but now it seems to work quite well.
fsnotify · May 1, 2024 · fd34385 · fd34385
1 parent 53b06a8
commit fd34385
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 75 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,9 +23,7 @@ jobs:
           command: |
             uname -a
             go version
-            FSNOTIFY_BUFFER=4096 go test -parallel 1 -race    ./...
-                                 go test -parallel 1 -race    ./...
-            FSNOTIFY_DEBUG=1     go test -parallel 1 -race -v ./...
+            ./run-ci
 
   # iOS
   ios:
@@ -50,8 +48,7 @@ jobs:
             export PATH=$PATH:/usr/local/Cellar/go/*/bin
             uname -a
             go version
-            FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
-                                 go test -parallel 1 -race ./...
+            ./run-ci
 
   # This is just Linux x86_64; also need to get a Go with GOOS=android, but
   # there aren't any pre-built versions of that on the Go site. Idk, disable for
@@ -79,6 +76,5 @@ jobs:
   #           uname -a
   #           export PATH=/usr/local/go/bin:$PATH
   #           go version
-  #           FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
-  #                                go test -parallel 1 -race ./...
+  #           ./run-ci
   #
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -9,6 +9,4 @@ freebsd_task:
       # run tests as user "cirrus" instead of root
     - pw useradd cirrus -m
     - chown -R cirrus:cirrus .
-    - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race    ./...
-    -                      sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race    ./...
-    - FSNOTIFY_DEBUG=1     sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race -v ./...
+    - sudo -u cirrus ./run-ci
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -40,6 +40,6 @@ jobs:
                 esac
             esac
 
-            go test -c
-            go build ./cmd/fsnotify
+            go test -c || exit 1
+            go build ./cmd/fsnotify || exit 1
           done
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,7 +3,7 @@ on:
   pull_request:
     paths: ['**.go', 'go.mod', '.github/workflows/*']
   push:
-    branches: ['main', 'aix']
+    paths: ['**.go', 'go.mod', '.github/workflows/*']
 
 jobs:
   linux:
@@ -18,11 +18,8 @@ jobs:
       - uses: 'actions/setup-go@v5'
         with:
           go-version: '${{ matrix.go }}'
-      - name: test
-        run: |
-          FSNOTIFY_BUFFER=4096 go test -parallel 1 -race    ./...
-                               go test -parallel 1 -race    ./...
-          FSNOTIFY_DEBUG=1     go test -parallel 1 -race -v ./...
+      - name: 'test'
+        run: './run-ci'
 
   windows:
     strategy:
@@ -36,15 +33,8 @@ jobs:
       - uses: 'actions/setup-go@v5'
         with:
           go-version: '${{ matrix.go }}'
-      - name: test
-        run: |
-          go test -parallel 1 -race    ./...
-
-          $Env:FSNOTIFY_BUFFER = 4096
-          go test -parallel 1 -race    ./...
-
-          $Env:FSNOTIFY_DEBUG = 1
-          go test -parallel 1 -race -v ./...
+      - name: 'test'
+        run: 'bash ./run-ci'
 
   # Test gccgo
   # Needs gccgo 13, which should be available in Ubuntu 24.04.
@@ -65,7 +55,7 @@ jobs:
   # it works on Windows and Linux with Go 1.17, then it probably does on macOS
   # too.
   macos:
-    name: test
+    name: 'test'
     strategy:
       fail-fast: false
       matrix:
@@ -76,19 +66,10 @@ jobs:
       - uses: 'actions/setup-go@v5'
         with:
           go-version: '1.22'
-      - name: test
-        run: |
-          FSNOTIFY_BUFFER=4096 go test -parallel 1 -race    ./...
-                               go test -parallel 1 -race    ./...
-          FSNOTIFY_DEBUG=1     go test -parallel 1 -race -v ./...
+      - name: 'test'
+        run: './run-ci'
 
-  # OpenBSD; no -race as the VM doesn't include the comp set.
-  #
-  # TODO: should probably add this, but on my local machine the tests time out
-  #       with -race as the waits aren't long enough (OpenBSD is kind of slow),
-  #       so should probably look into that first. Go 1.19 is supposed to have a
-  #       much faster race detector, so maybe waiting until we have that is
-  #       enough.
+  # OpenBSD
   openbsd:
     runs-on: 'ubuntu-latest'
     name: 'test (openbsd, 1.17)'
@@ -98,12 +79,12 @@ jobs:
         id:   'openbsd'
         uses: 'vmactions/openbsd-vm@v1'
         with:
-          prepare: pkg_add go
+          prepare: 'pkg_add go'
           run: |
             useradd -mG wheel action
-            FSNOTIFY_BUFFER=4096 su action -c 'go test -parallel 1    ./...'
-                                 su action -c 'go test -parallel 1    ./...'
-            FSNOTIFY_DEBUG=1     su action -c 'go test -parallel 1 -v ./...'
+            export GO_NORACE=1
+            export GO_NOPARALLEL=1
+            su action -c './run-ci'
 
   # NetBSD
   netbsd:
@@ -115,13 +96,11 @@ jobs:
         id:   'netbsd'
         uses: 'vmactions/netbsd-vm@v1'
         with:
-          prepare: pkg_add go
-          # TODO: no -race for the same reason as OpenBSD (the timing; it does run).
+          prepare: 'pkg_add go'
           run: |
             useradd -mG wheel action
-            FSNOTIFY_BUFFER=4096 su action -c '/usr/pkg/bin/go??? test -parallel 1    ./...'
-                                 su action -c '/usr/pkg/bin/go??? test -parallel 1    ./...'
-            FSNOTIFY_DEBUG=1     su action -c '/usr/pkg/bin/go??? test -parallel 1 -v ./...'
+            export GO=$(echo /usr/pkg/bin/go???)
+            su action -c './run-ci'
 
   # DragonFlyBSD
   dragonflybsd:
@@ -133,13 +112,11 @@ jobs:
         id:   'dragonflybsd'
         uses: 'vmactions/dragonflybsd-vm@v1'
         with:
-          prepare: pkg install -y go
-          # TODO: no -race for the same reason as OpenBSD (the timing; it does run).
+          prepare: 'pkg install -y go'
           run: |
             pw user add -mG wheel -n action
-            env FSNOTIFY_BUFFER=4096 su action -c 'go test -parallel 1    ./...'
-                                     su action -c 'go test -parallel 1    ./...'
-            env FSNOTIFY_DEBUG=1     su action -c 'go test -parallel 1 -v ./...'
+            export GO_NORACE=1
+            su action -c './run-ci'
 
   # illumos
   illumos:
@@ -156,9 +133,9 @@ jobs:
           useradd action
           export GOCACHE=/tmp/go-cache
           export GOPATH=/tmp/go-path
-          FSNOTIFY_BUFFER=4096 su action -c '/opt/ooce/go-1.22/bin/go test -parallel 1    ./...'
-                               su action -c '/opt/ooce/go-1.22/bin/go test -parallel 1    ./...'
-          FSNOTIFY_DEBUG=1     su action -c '/opt/ooce/go-1.22/bin/go test -parallel 1 -v ./...'
+          export GO=/opt/ooce/go-1.22/bin/go
+          export GO_NORACE=1
+          su action -c './run-ci'
 
 
   # Solaris
@@ -178,6 +155,4 @@ jobs:
   #         useradd action
   #         export GOCACHE=/tmp/go-cache
   #         export GOPATH=/tmp/go-path
-  #         FSNOTIFY_BUFFER=4096 su action -c 'go test -parallel 1    ./...'
-  #                              su action -c 'go test -parallel 1    ./...'
-  #         FSNOTIFY_DEBUG=1     su action -c 'go test -parallel 1 -v ./...'
+  #         su action -c './run-ci'
diff --git a/fsnotify_test.go b/fsnotify_test.go
@@ -80,11 +80,16 @@ func TestWatchMultipleWrite(t *testing.T) {
 	if err := fp.Close(); err != nil {
 		t.Fatal(err)
 	}
+	eventSeparator()
+
+	ev := w.stop(t)
+	// This can appear as one or two write events.
+	if len(ev) == 1 {
+		cmpEvents(t, tmp, ev, newEvents(t, `write /file`))
+	} else {
+		cmpEvents(t, tmp, ev, newEvents(t, "write /file\nwrite /file"))
+	}
 
-	cmpEvents(t, tmp, w.stop(t), newEvents(t, `
-		write  /file  # write X
-		write  /file  # write Y
-	`))
 }
 
 // Remove watched file with open fd
@@ -149,17 +154,17 @@ func TestWatchRemoveWatchedDir(t *testing.T) {
 
 	if runtime.GOOS != "windows" {
 		cmpEvents(t, tmp, w.stop(t), newEvents(t, `
-				remove    /
-				remove    /a
-				remove    /b
-				remove    /c
-				remove    /d
-				remove    /e
-				remove    /f
-				remove    /g
-				remove    /h
-				remove    /i
-				remove    /j`))
+			remove    /
+			remove    /a
+			remove    /b
+			remove    /c
+			remove    /d
+			remove    /e
+			remove    /f
+			remove    /g
+			remove    /h
+			remove    /i
+			remove    /j`))
 		return
 	}
 

diff --git a/run-ci b/run-ci
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -euC
+
+go=${GO:-go}
+race='-race'
+par=
+[ -n "${GO_NORACE:-}" ]     && race=
+[ -n "${GO_NOPARALLEL:-}" ] && par='-parallel=1'
+
+# Run tests once for quick exit if something is just wrong; then run it ten
+# times to catch flaky tests.
+set -x
+                         $go test $race $par -count=1  ./... || exit 1
+env FSNOTIFY_BUFFER=4096 $go test $race $par           ./... || exit 1
+                         $go test $race $par -count=10 ./... || exit 1
diff --git a/testdata/watch-recurse/remove-watched-dir b/testdata/watch-recurse/remove-watched-dir
@@ -20,6 +20,8 @@ mkdir /watch/j/a
 watch /watch/...
 rm -r /watch
 
+# TODO: this can be flaky on Windows; as sometimes a directory entry is missing.
+# Maybe we should add some "allow 2 missing" flag or something.
 Output:
 	remove      /watch/a
 	remove      /watch/b