-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[TT-744] Update to stream failure data and handle panics better (#141)
- Loading branch information
Showing
17 changed files
with
1,597 additions
and
755 deletions.
There are no files selected for viewing
158 changes: 66 additions & 92 deletions
158
go/go-test-results-parsing/__tests__/__snapshots__/main.test.ts.snap
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,102 +1,76 @@ | ||
| // Jest Snapshot v1, https://goo.gl/fbAQLP | ||
|
|
||
| exports[`output testing can deserialize file 1`] = ` | ||
| [ | ||
| { | ||
| "Action": "start", | ||
| }, | ||
| { | ||
| "Action": "pause", | ||
| }, | ||
| { | ||
| "Action": "cont", | ||
| }, | ||
| { | ||
| "Action": "run", | ||
| "Test": "TestOCIChart", | ||
| }, | ||
| { | ||
| "Action": "output", | ||
| "Output": "=== RUN TestOCIChart | ||
| ", | ||
| "Test": "TestOCIChart", | ||
| }, | ||
| { | ||
| "Action": "output", | ||
| "Output": "=== RUN Test5NodesPlus2MiningGethsReorgEnv | ||
| ", | ||
| "Test": "Test5NodesPlus2MiningGethsReorgEnv", | ||
| }, | ||
| { | ||
| "Action": "output", | ||
| "Output": "blah blah blah | ||
| ", | ||
| }, | ||
| { | ||
| "Action": "skip", | ||
| "Elapsed": 1, | ||
| "Test": "TestMinResources5NodesEnvWithBlockscoutSkip", | ||
| }, | ||
| { | ||
| "Action": "pass", | ||
| "Elapsed": 506.4, | ||
| "Test": "TestOCIChart", | ||
| }, | ||
| { | ||
| "Action": "fail", | ||
| "Elapsed": 522.96, | ||
| "Test": "Test5NodesPlus2MiningGethsReorgEnv", | ||
| }, | ||
| { | ||
| "Action": "fail", | ||
| "Elapsed": 551.993, | ||
| }, | ||
| ] | ||
| exports[`output testing can parse a panic for a test name and only output logs for that panic from that package 1`] = ` | ||
| "github.com/smartcontractkit/chainlink/v2/core/services/telemetry has failure logging but no test failures, the output below may be useful for triage | ||
| === RUN TestNewManager | ||
| --- PASS: TestNewManager (0.00s) | ||
| PASS | ||
| panic: Log in goroutine after TestNewManager has completed: 2023-11-28T11:38:06.521Z WARN TelemetryManager.TelemetryIngressBatchClient wsrpc@v0.7.2/uni_client.go:97 ctx error context canceled reconnecting {"version": "2.7.0@0957729"} | ||
| goroutine 80 [running]: | ||
| testing.(*common).logDepth(0xc00172c000, {0xc000052840, 0xad}, 0x3) | ||
| /opt/hostedtoolcache/go/1.21.4/x64/src/testing/testing.go:1022 +0x4c5 | ||
| testing.(*common).log(...) | ||
| /opt/hostedtoolcache/go/1.21.4/x64/src/testing/testing.go:1004 | ||
| testing.(*common).Logf(0xc00172c000, {0x18770d6?, 0x4110c5?}, {0xc0016a1190?, 0x15d9f00?, 0x1?}) | ||
| /opt/hostedtoolcache/go/1.21.4/x64/src/testing/testing.go:1055 +0x54 | ||
| go.uber.org/zap/zaptest.testingWriter.Write({{0x7f63c4847198?, 0xc00172c000?}, 0x70?}, {0xc0017aa800?, 0xae, 0xc0016a1180?}) | ||
| /home/runner/go/pkg/mod/go.uber.org/zap@v1.26.0/zaptest/logger.go:130 +0xdc | ||
| go.uber.org/zap/zapcore.(*ioCore).Write(0xc0017808d0, {0x1, {0xc15192279f1426a5, 0x9fcc1ad, 0x2bb1d20}, {0xc001080060, 0x2c}, {0xc001080270, 0x27}, {0x1, ...}, ...}, ...) | ||
| /home/runner/go/pkg/mod/go.uber.org/zap@v1.26.0/zapcore/core.go:99 +0xb5 | ||
| go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0010bc820, {0x0, 0x0, 0x0}) | ||
| /home/runner/go/pkg/mod/go.uber.org/zap@v1.26.0/zapcore/entry.go:253 +0x1dc | ||
| go.uber.org/zap.(*SugaredLogger).log(0xc000246168, 0x1, {0x197ac35?, 0x19?}, {0xc0016a1140?, 0x1?, 0x1?}, {0x0, 0x0, 0x0}) | ||
| /home/runner/go/pkg/mod/go.uber.org/zap@v1.26.0/sugar.go:316 +0xec | ||
| go.uber.org/zap.(*SugaredLogger).Warnf(...) | ||
| FAIL github.com/smartcontractkit/chainlink/v2/core/services/telemetry 0.192s | ||
| " | ||
| `; | ||
| exports[`output testing failed test list 1`] = ` | ||
| [ | ||
| "Test5NodesPlus2MiningGethsReorgEnv", | ||
| ] | ||
| exports[`output testing can read file with all failures 1`] = ` | ||
| "=== RUN TestGetImage | ||
| mirror_test.go:12: | ||
| Error Trace: /Users/blarg/git/chainlink-testing-framework/mirror/mirror_test.go:12 | ||
| Error: An error is expected but got nil. | ||
| Test: TestGetImage | ||
| --- FAIL: TestGetImage (0.00s) | ||
| FAIL | ||
| FAIL github.com/smartcontractkit/chainlink-testing-framework/mirror 0.349s | ||
| " | ||
| `; | ||
| exports[`output testing failure without a test failure 1`] = `[]`; | ||
| exports[`output testing can read file with mix of pass and failures 1`] = ` | ||
| "=== RUN TestFailTest | ||
| mirror_test.go:12: | ||
| Error Trace: /Users/blarg/git/chainlink-testing-framework/failpackage/mirror_test.go:12 | ||
| Error: An error is expected but got nil. | ||
| Test: TestFailTest | ||
| --- FAIL: TestFailTest (0.00s) | ||
| FAIL | ||
| FAIL github.com/smartcontractkit/chainlink-testing-framework/failpackage 0.349s | ||
| " | ||
| `; | ||
| exports[`output testing json test output 1`] = ` | ||
| [ | ||
| { | ||
| "Action": "start", | ||
| }, | ||
| { | ||
| "Action": "output", | ||
| "Output": "=== RUN Test5NodesPlus2MiningGethsReorgEnv | ||
| ", | ||
| "Test": "Test5NodesPlus2MiningGethsReorgEnv", | ||
| }, | ||
| { | ||
| "Action": "output", | ||
| "Output": "blah blah blah | ||
| ", | ||
| }, | ||
| { | ||
| "Action": "fail", | ||
| "Elapsed": 522.96, | ||
| "Test": "Test5NodesPlus2MiningGethsReorgEnv", | ||
| }, | ||
| { | ||
| "Action": "fail", | ||
| "Elapsed": 551.993, | ||
| }, | ||
| ] | ||
| exports[`output testing can read file with mix of pass and failures failures and non json txt injected from other potential errors in the go runner 1`] = ` | ||
| "some error output stuff that won't parse correctly so just write it out1 | ||
| some error output stuff that won't parse correctly so just write it out2 | ||
| === RUN TestFailTest | ||
| mirror_test.go:12: | ||
| Error Trace: /Users/blarg/git/chainlink-testing-framework/failpackage/mirror_test.go:12 | ||
| Error: An error is expected but got nil. | ||
| Test: TestFailTest | ||
| --- FAIL: TestFailTest (0.00s) | ||
| FAIL | ||
| FAIL github.com/smartcontractkit/chainlink-testing-framework/failpackage 0.349s | ||
| some error output stuff that won't parse correctly so just write it out3 | ||
| " | ||
| `; | ||
| exports[`output testing standard test output 1`] = ` | ||
| [ | ||
| { | ||
| "Action": "output", | ||
| "Output": "=== RUN Test5NodesPlus2MiningGethsReorgEnv | ||
| ", | ||
| "Test": "Test5NodesPlus2MiningGethsReorgEnv", | ||
| }, | ||
| ] | ||
| exports[`output testing can read file with package failure but no test failure 1`] = ` | ||
| "github.com/smartcontractkit/chainlink-testing-framework/mirror has failure logging but no test failures, the output below may be useful for triage | ||
| === RUN TestGetImage | ||
| --- PASS: TestGetImage (0.00s) | ||
| FAIL | ||
| FAIL github.com/smartcontractkit/chainlink-testing-framework/mirror 0.349s | ||
| " | ||
| `; |
14 changes: 14 additions & 0 deletions
14
go/go-test-results-parsing/__tests__/fixtures/all_failing.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| {"Time":"2023-11-27T15:43:53.048232-07:00","Action":"start","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror"} | ||
| {"Time":"2023-11-27T15:43:53.397883-07:00","Action":"pause","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:43:53.397884-07:00","Action":"cont","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:43:53.397885-07:00","Action":"run","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:43:53.397999-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"=== RUN TestGetImage\n"} | ||
| {"Time":"2023-11-27T15:43:53.398176-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":" mirror_test.go:12: \n"} | ||
| {"Time":"2023-11-27T15:43:53.398181-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":" \tError Trace:\t/Users/blarg/git/chainlink-testing-framework/mirror/mirror_test.go:12\n"} | ||
| {"Time":"2023-11-27T15:43:53.398191-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":" \tError: \tAn error is expected but got nil.\n"} | ||
| {"Time":"2023-11-27T15:43:53.398197-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":" \tTest: \tTestGetImage\n"} | ||
| {"Time":"2023-11-27T15:43:53.398634-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"--- FAIL: TestGetImage (0.00s)\n"} | ||
| {"Time":"2023-11-27T15:43:53.39865-07:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Elapsed":0} | ||
| {"Time":"2023-11-27T15:43:53.398988-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"FAIL\n"} | ||
| {"Time":"2023-11-27T15:43:53.399042-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"FAIL\tgithub.com/smartcontractkit/chainlink-testing-framework/mirror\t0.349s\n"} | ||
| {"Time":"2023-11-27T15:43:53.399052-07:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Elapsed":0.351} |
10 changes: 10 additions & 0 deletions
10
go/go-test-results-parsing/__tests__/fixtures/all_passing.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| {"Time":"2023-11-27T15:39:38.891004-07:00","Action":"start","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror"} | ||
| {"Time":"2023-11-27T15:39:39.223129-07:00","Action":"pause","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:39:39.223129-07:00","Action":"cont","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:39:39.223129-07:00","Action":"run","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T15:39:39.223203-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"=== RUN TestGetImage\n"} | ||
| {"Time":"2023-11-27T15:39:39.223325-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"--- PASS: TestGetImage (0.00s)\n"} | ||
| {"Time":"2023-11-27T15:39:39.223335-07:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Elapsed":0} | ||
| {"Time":"2023-11-27T15:39:39.223392-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"PASS\n"} | ||
| {"Time":"2023-11-27T15:39:39.223823-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"ok \tgithub.com/smartcontractkit/chainlink-testing-framework/mirror\t0.332s\n"} | ||
| {"Time":"2023-11-27T15:39:39.223871-07:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Elapsed":0.333} |
8 changes: 8 additions & 0 deletions
8
go/go-test-results-parsing/__tests__/fixtures/failing_package_without_test_fail.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| {"Time":"2023-11-27T16:16:06.849991-07:00","Action":"start","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror"} | ||
| {"Time":"2023-11-27T16:16:07.184219-07:00","Action":"run","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage"} | ||
| {"Time":"2023-11-27T16:16:07.184264-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"=== RUN TestGetImage\n"} | ||
| {"Time":"2023-11-27T16:16:07.18478-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Output":"--- PASS: TestGetImage (0.00s)\n"} | ||
| {"Time":"2023-11-27T16:16:07.184798-07:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Test":"TestGetImage","Elapsed":0} | ||
| {"Time":"2023-11-27T15:43:53.398988-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"FAIL\n"} | ||
| {"Time":"2023-11-27T15:43:53.399042-07:00","Action":"output","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Output":"FAIL\tgithub.com/smartcontractkit/chainlink-testing-framework/mirror\t0.349s\n"} | ||
| {"Time":"2023-11-27T16:16:07.185307-07:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink-testing-framework/mirror","Elapsed":0.335} |
4 changes: 0 additions & 4 deletions
4
go/go-test-results-parsing/__tests__/fixtures/failure_output_without_test_failure.json
This file was deleted.
Oops, something went wrong.
11 changes: 0 additions & 11 deletions
11
go/go-test-results-parsing/__tests__/fixtures/go_test_results_input.json
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.