From 59fdb69b9a7d0ac2c237b4029b3b8c92d93f7339 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 7 Feb 2022 13:36:42 +0300 Subject: [PATCH 1/9] cli: add temp workaround to handle absolute log path for Windows Fix #2269. --- cli/server/server.go | 43 +++++++++++++++++++++++++++++++++++++++ cli/server/server_test.go | 1 - 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/cli/server/server.go b/cli/server/server.go index 19d3a407a9..36bfd1c477 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "net/url" "os" "os/signal" + "runtime" "syscall" "github.com/nspcc-dev/neo-go/cli/options" @@ -29,6 +31,10 @@ import ( "go.uber.org/zap/zapcore" ) +// _winfileSinkRegistered denotes whether zap has registered +// user-supplied factory for all sinks with `winfile`-prefixed scheme. +var _winfileSinkRegistered bool + // NewCommands returns 'node' command. func NewCommands() []cli.Command { var cfgFlags = []cli.Flag{ @@ -145,6 +151,43 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) return nil, err } + if runtime.GOOS == "windows" { + if !_winfileSinkRegistered { + // See https://github.com/uber-go/zap/issues/621. + err := zap.RegisterSink("winfile", func(u *url.URL) (zap.Sink, error) { + if u.User != nil { + return nil, fmt.Errorf("user and password not allowed with file URLs: got %v", u) + } + if u.Fragment != "" { + return nil, fmt.Errorf("fragments not allowed with file URLs: got %v", u) + } + if u.RawQuery != "" { + return nil, fmt.Errorf("query parameters not allowed with file URLs: got %v", u) + } + // Error messages are better if we check hostname and port separately. + if u.Port() != "" { + return nil, fmt.Errorf("ports not allowed with file URLs: got %v", u) + } + if hn := u.Hostname(); hn != "" && hn != "localhost" { + return nil, fmt.Errorf("file URLs must leave host empty or use localhost: got %v", u) + } + switch u.Path { + case "stdout": + return os.Stdout, nil + case "stderr": + return os.Stderr, nil + } + return os.OpenFile(u.Path[1:], // Remove leading slash left after url.Parse. + os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + }) + if err != nil { + return nil, fmt.Errorf("failed to register windows-specific sinc: %w", err) + } + _winfileSinkRegistered = true + } + logPath = "winfile:///" + logPath + } + cc.OutputPaths = []string{logPath} } diff --git a/cli/server/server_test.go b/cli/server/server_test.go index 7d0e8ebcea..3165e7037d 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -41,7 +41,6 @@ func TestGetConfigFromContext(t *testing.T) { } func TestHandleLoggingParams(t *testing.T) { - // This test is failing on Windows, see https://github.com/nspcc-dev/neo-go/issues/2269 d := t.TempDir() testLog := filepath.Join(d, "file.log") From 2736d62d471ca1275161501081f262073840782e Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 7 Feb 2022 18:51:54 +0300 Subject: [PATCH 2/9] cli: ensure opened sinks are properly closed zap never closes open sinks except its own tests. This behaviour prevents TestHandleLoggingParams from successful cleanup because temp log output file can't be closed due to the following error: ``` TempDir RemoveAll cleanup: remove C:\\Users\\Anna\\AppData\\Local\\Temp\\TestHandleLoggingParams_debug5796883 33\\001\\file.log: The process cannot access the file because it is being used by another process. ``` So this tremendous cludge is made mosetly for our testing code. It is not for concurrent usage (we don't have cases of multithreaded access to logger output sink). --- cli/server/server.go | 42 +++++++++++++++++++++++++++++---------- cli/server/server_test.go | 24 ++++++++++++++++++---- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 36bfd1c477..310fada081 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -31,9 +31,12 @@ import ( "go.uber.org/zap/zapcore" ) -// _winfileSinkRegistered denotes whether zap has registered -// user-supplied factory for all sinks with `winfile`-prefixed scheme. -var _winfileSinkRegistered bool +var ( + // _winfileSinkRegistered denotes whether zap has registered + // user-supplied factory for all sinks with `winfile`-prefixed scheme. + _winfileSinkRegistered bool + _winfileSinkCloser func() error +) // NewCommands returns 'node' command. func NewCommands() []cli.Command { @@ -130,7 +133,9 @@ func getConfigFromContext(ctx *cli.Context) (config.Config, error) { // handleLoggingParams reads logging parameters. // If user selected debug level -- function enables it. // If logPath is configured -- function creates dir and file for logging. -func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) (*zap.Logger, error) { +// If logPath is configured on Windows -- function returns closer to be +// able to close sink for opened log output file. +func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) (*zap.Logger, func() error, error) { level := zapcore.InfoLevel if ctx.Bool("debug") { level = zapcore.DebugLevel @@ -148,7 +153,7 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) if logPath := cfg.LogPath; logPath != "" { if err := io.MakeDirForFile(logPath, "logger"); err != nil { - return nil, err + return nil, nil, err } if runtime.GOOS == "windows" { @@ -177,11 +182,16 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) case "stderr": return os.Stderr, nil } - return os.OpenFile(u.Path[1:], // Remove leading slash left after url.Parse. + f, err := os.OpenFile(u.Path[1:], // Remove leading slash left after url.Parse. os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + _winfileSinkCloser = func() error { + _winfileSinkCloser = nil + return f.Close() + } + return f, err }) if err != nil { - return nil, fmt.Errorf("failed to register windows-specific sinc: %w", err) + return nil, nil, fmt.Errorf("failed to register windows-specific sinc: %w", err) } _winfileSinkRegistered = true } @@ -191,7 +201,8 @@ func handleLoggingParams(ctx *cli.Context, cfg config.ApplicationConfiguration) cc.OutputPaths = []string{logPath} } - return cc.Build() + log, err := cc.Build() + return log, _winfileSinkCloser, err } func initBCWithMetrics(cfg config.Config, log *zap.Logger) (*core.Blockchain, *metrics.Service, *metrics.Service, error) { @@ -215,10 +226,13 @@ func dumpDB(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } - log, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } + if logCloser != nil { + defer func() { _ = logCloser() }() + } count := uint32(ctx.Uint("count")) start := uint32(ctx.Uint("start")) @@ -262,10 +276,13 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return err } - log, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } + if logCloser != nil { + defer func() { _ = logCloser() }() + } count := uint32(ctx.Uint("count")) var inStream = os.Stdin @@ -438,10 +455,13 @@ func startServer(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } - log, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + log, logCloser, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) if err != nil { return cli.NewExitError(err, 1) } + if logCloser != nil { + defer func() { _ = logCloser() }() + } grace, cancel := context.WithCancel(newGraceContext()) defer cancel() diff --git a/cli/server/server_test.go b/cli/server/server_test.go index 3165e7037d..88909d8644 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -52,8 +52,9 @@ func TestHandleLoggingParams(t *testing.T) { cfg := config.ApplicationConfiguration{ LogPath: filepath.Join(logfile, "file.log"), } - _, err := handleLoggingParams(ctx, cfg) + _, closer, err := handleLoggingParams(ctx, cfg) require.Error(t, err) + require.Nil(t, closer) }) t.Run("default", func(t *testing.T) { @@ -62,8 +63,13 @@ func TestHandleLoggingParams(t *testing.T) { cfg := config.ApplicationConfiguration{ LogPath: testLog, } - logger, err := handleLoggingParams(ctx, cfg) + logger, closer, err := handleLoggingParams(ctx, cfg) require.NoError(t, err) + t.Cleanup(func() { + if closer != nil { + require.NoError(t, closer()) + } + }) require.True(t, logger.Core().Enabled(zap.InfoLevel)) require.False(t, logger.Core().Enabled(zap.DebugLevel)) }) @@ -75,8 +81,13 @@ func TestHandleLoggingParams(t *testing.T) { cfg := config.ApplicationConfiguration{ LogPath: testLog, } - logger, err := handleLoggingParams(ctx, cfg) + logger, closer, err := handleLoggingParams(ctx, cfg) require.NoError(t, err) + t.Cleanup(func() { + if closer != nil { + require.NoError(t, closer()) + } + }) require.True(t, logger.Core().Enabled(zap.InfoLevel)) require.True(t, logger.Core().Enabled(zap.DebugLevel)) }) @@ -95,8 +106,13 @@ func TestInitBCWithMetrics(t *testing.T) { ctx := cli.NewContext(cli.NewApp(), set, nil) cfg, err := getConfigFromContext(ctx) require.NoError(t, err) - logger, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) + logger, closer, err := handleLoggingParams(ctx, cfg.ApplicationConfiguration) require.NoError(t, err) + t.Cleanup(func() { + if closer != nil { + require.NoError(t, closer()) + } + }) t.Run("bad store", func(t *testing.T) { _, _, _, err = initBCWithMetrics(config.Config{}, logger) From 8ff7cd865d532168a8e89ca6821ceab914ccebe0 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 8 Feb 2022 14:02:46 +0300 Subject: [PATCH 3/9] cli: ensure chain is properly stopped after chain-related commands Blockchain occupies resources (e.g. it opens log files for DB, etc.) on creation and running. We need to release these resources if something goes wrong during execution chain-related commands. This commit solves the following problem on Windows: ``` --- FAIL: TestServerStart (0.32s) --- FAIL: TestServerStart/stateroot_service_is_on_&&_StateRootInHeader=true (0.04s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_stateroot_service_is_on_&&_StateRootInHeader=true460557297\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. --- FAIL: TestServerStart/invalid_Oracle_config (0.03s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_invalid_Oracle_config810064028\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. --- FAIL: TestServerStart/invalid_consensus_config (0.04s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_invalid_consensus_config217270091\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. --- FAIL: TestServerStart/invalid_Notary_config (0.07s) --- FAIL: TestServerStart/invalid_Notary_config/malformed_config (0.04s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_invalid_Notary_config_malformed_config754934830\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. --- FAIL: TestServerStart/invalid_Notary_config/invalid_wallet (0.03s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_invalid_Notary_config_invalid_wallet934249397\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. --- FAIL: TestServerStart/good (0.11s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_good596150160\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. ``` This commit also unifies blockchain and services releasing code. --- cli/server/server.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 310fada081..885c0bf4f1 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -304,9 +304,11 @@ func restoreDB(ctx *cli.Context) error { if err != nil { return err } - defer chain.Close() - defer prometheus.ShutDown() - defer pprof.ShutDown() + defer func() { + pprof.ShutDown() + prometheus.ShutDown() + chain.Close() + }() var start uint32 if ctx.Bool("incremental") { @@ -472,6 +474,11 @@ func startServer(ctx *cli.Context) error { if err != nil { return cli.NewExitError(err, 1) } + defer func() { + pprof.ShutDown() + prometheus.ShutDown() + chain.Close() + }() serv, err := network.NewServer(serverConfig, chain, chain.GetStateSyncModule(), log) if err != nil { @@ -534,9 +541,6 @@ Main: if serverErr := rpcServer.Shutdown(); serverErr != nil { shutdownErr = fmt.Errorf("error on shutdown: %w", serverErr) } - prometheus.ShutDown() - pprof.ShutDown() - chain.Close() break Main } } From d73f95e9884fc1d1f3a0d24462fa45fce563d71d Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 8 Feb 2022 17:03:17 +0300 Subject: [PATCH 4/9] cli: do not run test server on windows Currently we can't properly stop running server on Windows and SIGHUP is also not supported. This leads to occupied resources and failed test cleanup: ``` --- FAIL: TestServerStart (0.35s) --- FAIL: TestServerStart/good (0.10s) testing.go:894: TempDir RemoveAll cleanup: remove C:\Users\Anna\AppData\Local\Temp\TestServerStart_good337747932\001\neogotestchain\000001.log: The process cannot access the file because it is being used by another process. 2022-02-08T14:11:20.959+0300 INFO persisted to disk {"blocks": 0, "keys": 112, "headerHeight": 0, "blockHeight": 0, "took": "10.0049ms"} ``` --- cli/server_test.go | 50 +++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/cli/server_test.go b/cli/server_test.go index 308ce98fcc..6b6415e017 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -1,11 +1,11 @@ package main import ( - "fmt" "io" "io/ioutil" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -96,29 +96,33 @@ func TestServerStart(t *testing.T) { e.RunWithError(t, baseCmd...) }) }) - t.Run("good", func(t *testing.T) { - saveCfg(t, func(cfg *config.Config) {}) + // We can't properly shutdown server on windows and release the resources. + // Also, windows doesn't support SIGHUP and SIGINT. + if runtime.GOOS != "windows" { + t.Run("good", func(t *testing.T) { + saveCfg(t, func(cfg *config.Config) {}) - go func() { - e.Run(t, baseCmd...) - }() + go func() { + e.Run(t, baseCmd...) + }() - var line string - require.Eventually(t, func() bool { - line, err = e.Out.ReadString('\n') - if err != nil && err != io.EOF { - t.Fatalf(fmt.Sprintf("unexpected error while reading CLI output: %s", err)) + var line string + require.Eventually(t, func() bool { + line, err = e.Out.ReadString('\n') + if err != nil && err != io.EOF { + t.Fatalf("unexpected error while reading CLI output: %s", err) + } + return err == nil + }, 2*time.Second, 100*time.Millisecond) + lines := strings.Split(server.Logo(), "\n") + for _, expected := range lines { + // It should be regexp, so escape all backslashes. + expected = strings.ReplaceAll(expected, `\`, `\\`) + e.checkLine(t, line, expected) + line = e.getNextLine(t) } - return err == nil - }, 2*time.Second, 100*time.Millisecond) - lines := strings.Split(server.Logo(), "\n") - for _, expected := range lines { - // It should be regexp, so escape all backslashes. - expected = strings.ReplaceAll(expected, `\`, `\\`) - e.checkLine(t, line, expected) - line = e.getNextLine(t) - } - e.checkNextLine(t, "") - e.checkEOF(t) - }) + e.checkNextLine(t, "") + e.checkEOF(t) + }) + } } From a9872063fed27a7705745cdf9073a1032c22a0d7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 9 Feb 2022 16:36:27 +0300 Subject: [PATCH 5/9] vm: refactor VM CLI tests as they take too long to pass These tests are slow on Windows, so refactor them a bit and avoid the following error: ``` --- FAIL: TestRunWithDifferentArguments (4.01s) cli_test.go:96: Error Trace: cli_test.go:96 cli_test.go:321 Error: command took too long time Test: TestRunWithDifferentArguments ``` --- pkg/vm/cli/cli_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/vm/cli/cli_test.go b/pkg/vm/cli/cli_test.go index 28cda13a8a..8c71a30fdf 100644 --- a/pkg/vm/cli/cli_test.go +++ b/pkg/vm/cli/cli_test.go @@ -84,6 +84,10 @@ func newTestVMCLIWithLogo(t *testing.T, printLogo bool) *executor { } func (e *executor) runProg(t *testing.T, commands ...string) { + e.runProgWithTimeout(t, 4*time.Second, commands...) +} + +func (e *executor) runProgWithTimeout(t *testing.T, timeout time.Duration, commands ...string) { cmd := strings.Join(commands, "\n") + "\n" e.in.WriteString(cmd + "\n") go func() { @@ -92,7 +96,7 @@ func (e *executor) runProg(t *testing.T, commands ...string) { }() select { case <-e.ch: - case <-time.After(4 * time.Second): + case <-time.After(timeout): require.Fail(t, "command took too long time") } } @@ -213,7 +217,7 @@ go 1.16`) require.NoError(t, ioutil.WriteFile(filepath.Join(tmpDir, "go.mod"), goMod, os.ModePerm)) e := newTestVMCLI(t) - e.runProg(t, + e.runProgWithTimeout(t, 10*time.Second, "loadgo", "loadgo "+filenameErr, "loadgo "+filename, @@ -318,7 +322,7 @@ func TestRunWithDifferentArguments(t *testing.T) { filename = "'" + filename + "'" e := newTestVMCLI(t) - e.runProg(t, + e.runProgWithTimeout(t, 30*time.Second, "loadgo "+filename, "run notexists", "loadgo "+filename, "run negate false", "loadgo "+filename, "run negate true", From 0de4c0fc17b01295874fd383cff6ce75d52373c0 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 9 Feb 2022 18:17:47 +0300 Subject: [PATCH 6/9] compiler: use full paths for debug info files if relative can't be constructed Firstly introduced in 9871dc8f5aa33c17651848df637ed1185f43aa18. It's OK if relative path can't be constructed e.g. for interop dependencies. Otherwice leads to failing tests on Windows with the following error: ``` 2022-02-09T09:39:31.3307626Z panic: Rel: can't make D:\a\neo-go\neo-go\pkg\interop\neogointernal\syscall.go relative to C:\Users\RUNNER~1\AppData\Local\Temp\TestContractInitAndCompile1998984267\001\testcontract [recovered] 2022-02-09T09:39:31.3308830Z panic: Rel: can't make D:\a\neo-go\neo-go\pkg\interop\neogointernal\syscall.go relative to C:\Users\RUNNER~1\AppData\Local\Temp\TestContractInitAndCompile1998984267\001\testcontract 2022-02-09T09:39:31.3309285Z 2022-02-09T09:39:31.3309390Z goroutine 302 [running]: 2022-02-09T09:39:31.3309725Z testing.tRunner.func1.2({0x14dbfe0, 0xc00051bdb0}) 2022-02-09T09:39:31.3310187Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/testing/testing.go:1209 +0x36c 2022-02-09T09:39:31.3310538Z testing.tRunner.func1() 2022-02-09T09:39:31.3310937Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/testing/testing.go:1212 +0x3b6 2022-02-09T09:39:31.3311285Z panic({0x14dbfe0, 0xc00051bdb0}) 2022-02-09T09:39:31.3311689Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/runtime/panic.go:1047 +0x266 2022-02-09T09:39:31.3312319Z github.com/nspcc-dev/neo-go/pkg/compiler.(*codegen).fillDocumentInfo.func1(0xc0002d0b40) 2022-02-09T09:39:31.3312885Z D:/a/neo-go/neo-go/pkg/compiler/analysis.go:240 +0x34d 2022-02-09T09:39:31.3313397Z go/token.(*FileSet).Iterate(0xc0000a69c0, 0xc0004898b0) 2022-02-09T09:39:31.3313839Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/go/token/position.go:463 +0xde 2022-02-09T09:39:31.3314404Z github.com/nspcc-dev/neo-go/pkg/compiler.(*codegen).fillDocumentInfo(0xc0002e4d20) 2022-02-09T09:39:31.3314947Z D:/a/neo-go/neo-go/pkg/compiler/analysis.go:236 +0x97 2022-02-09T09:39:31.3315530Z github.com/nspcc-dev/neo-go/pkg/compiler.(*codegen).compile(0xc0002e4d20, 0xc00009a690, 0xc000101540) 2022-02-09T09:39:31.3316064Z D:/a/neo-go/neo-go/pkg/compiler/codegen.go:2088 +0x192 2022-02-09T09:39:31.3316563Z github.com/nspcc-dev/neo-go/pkg/compiler.codeGen(0xc00009a690) 2022-02-09T09:39:31.3317066Z D:/a/neo-go/neo-go/pkg/compiler/codegen.go:2167 +0x7cd 2022-02-09T09:39:31.3317679Z github.com/nspcc-dev/neo-go/pkg/compiler.CompileWithOptions({0xc0000435e0, 0x62}, {0x0, 0x0}, 0xc0004bcf70) 2022-02-09T09:39:31.3318233Z D:/a/neo-go/neo-go/pkg/compiler/compiler.go:225 +0xde 2022-02-09T09:39:31.3318804Z github.com/nspcc-dev/neo-go/pkg/compiler.CompileAndSave({0xc0000435e0, 0x62}, 0xc0004bcf70) 2022-02-09T09:39:31.3319353Z D:/a/neo-go/neo-go/pkg/compiler/compiler.go:241 +0x3a5 2022-02-09T09:39:31.3319889Z github.com/nspcc-dev/neo-go/cli/smartcontract.contractCompile(0xc0004e42c0) 2022-02-09T09:39:31.3320456Z D:/a/neo-go/neo-go/cli/smartcontract/smart_contract.go:520 +0xc2d 2022-02-09T09:39:31.3320873Z github.com/urfave/cli.HandleAction({0x14b6fa0, 0x17b8d60}, 0x7) 2022-02-09T09:39:31.3321344Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:524 +0xf4 2022-02-09T09:39:31.3321931Z github.com/urfave/cli.Command.Run({{0x16460d2, 0x7}, {0x0, 0x0}, {0x0, 0x0, 0x0}, {0x166fe9e, 0x27}, {0x0, ...}, ...}, ...) 2022-02-09T09:39:31.3322443Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:173 +0xc09 2022-02-09T09:39:31.3322894Z github.com/urfave/cli.(*App).RunAsSubcommand(0xc0000b7340, 0xc0004e4000) 2022-02-09T09:39:31.3323375Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:405 +0x106c 2022-02-09T09:39:31.3323852Z github.com/urfave/cli.Command.startApp({{0x1647536, 0x8}, {0x0, 0x0}, {0x0, 0x0, 0x0}, {0x1671070, 0x28}, {0x0, ...}, ...}, ...) 2022-02-09T09:39:31.3324365Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:372 +0x108c 2022-02-09T09:39:31.3324838Z github.com/urfave/cli.Command.Run({{0x1647536, 0x8}, {0x0, 0x0}, {0x0, 0x0, 0x0}, {0x1671070, 0x28}, {0x0, ...}, ...}, ...) 2022-02-09T09:39:31.3325327Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:102 +0xe8c 2022-02-09T09:39:31.3325744Z github.com/urfave/cli.(*App).Run(0xc000323880, {0xc0004237a0, 0xb, 0x12}) 2022-02-09T09:39:31.3326198Z C:/Users/runneradmin/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:277 +0xa8c 2022-02-09T09:39:31.3326770Z github.com/nspcc-dev/neo-go/cli.(*executor).run(0xc0000709c0, {0xc0004237a0, 0xb, 0x12}) 2022-02-09T09:39:31.3327275Z D:/a/neo-go/neo-go/cli/executor_test.go:273 +0x3a5 2022-02-09T09:39:31.3327812Z github.com/nspcc-dev/neo-go/cli.(*executor).Run(0x14aa020, 0xc000493950, {0xc0004237a0, 0xb, 0x12}) 2022-02-09T09:39:31.3328324Z D:/a/neo-go/neo-go/cli/executor_test.go:263 +0x10f 2022-02-09T09:39:31.3328844Z github.com/nspcc-dev/neo-go/cli.TestContractInitAndCompile(0xc0001d69c0) 2022-02-09T09:39:31.3329377Z D:/a/neo-go/neo-go/cli/contract_test.go:155 +0x147f 2022-02-09T09:39:31.3329694Z testing.tRunner(0xc0001d69c0, 0x17b8be8) 2022-02-09T09:39:31.3330123Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/testing/testing.go:1259 +0x230 2022-02-09T09:39:31.3330470Z created by testing.(*T).Run 2022-02-09T09:39:31.3330862Z C:/hostedtoolcache/windows/go/1.17.6/x64/src/testing/testing.go:1306 +0x727 2022-02-09T09:39:31.3331325Z FAIL github.com/nspcc-dev/neo-go/cli 5.552s ``` --- pkg/compiler/analysis.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/compiler/analysis.go b/pkg/compiler/analysis.go index d901dfb978..845607f4da 100644 --- a/pkg/compiler/analysis.go +++ b/pkg/compiler/analysis.go @@ -235,9 +235,10 @@ func (c *codegen) fillDocumentInfo() { fset := c.buildInfo.config.Fset fset.Iterate(func(f *token.File) bool { filePath := f.Position(f.Pos(0)).Filename - filePath, err := filepath.Rel(c.buildInfo.config.Dir, filePath) - if err != nil { - panic(err) + rel, err := filepath.Rel(c.buildInfo.config.Dir, filePath) + // It's OK if we can't construct relative path, e.g. for interop dependencies. + if err == nil { + filePath = rel } c.docIndex[filePath] = len(c.documents) c.documents = append(c.documents, filePath) From 9bbd94d0fa0395a3b7472dfec72604833e0b99a1 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 9 Feb 2022 20:14:11 +0300 Subject: [PATCH 7/9] network: tune waiting limits in tests Some tests are failing on Windows due to slow runners with errors like the following: ``` 2022-02-09T17:11:20.3127016Z --- FAIL: TestGetData/transaction (1.82s) 2022-02-09T17:11:20.3127385Z server_test.go:500: 2022-02-09T17:11:20.3127878Z Error Trace: server_test.go:500 2022-02-09T17:11:20.3128533Z server_test.go:520 2022-02-09T17:11:20.3128978Z Error: Condition never satisfied 2022-02-09T17:11:20.3129479Z Test: TestGetData/transaction ``` --- pkg/network/server_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index d20d8a59ae..0624b157b2 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -404,7 +404,7 @@ func TestBlock(t *testing.T) { b := block.New(false) b.Index = 12345 s.testHandleMessage(t, nil, CMDBlock, b) - require.Eventually(t, func() bool { return s.chain.BlockHeight() == 12345 }, time.Second, time.Millisecond*500) + require.Eventually(t, func() bool { return s.chain.BlockHeight() == 12345 }, 2*time.Second, time.Millisecond*500) } func TestConsensus(t *testing.T) { @@ -497,8 +497,8 @@ func (s *Server) testHandleGetData(t *testing.T, invType payload.InventoryType, s.testHandleMessage(t, p, CMDGetData, payload.NewInventory(invType, hs)) - require.Eventually(t, func() bool { return recvResponse.Load() }, time.Second, time.Millisecond) - require.Eventually(t, func() bool { return recvNotFound.Load() }, time.Second, time.Millisecond) + require.Eventually(t, func() bool { return recvResponse.Load() }, 2*time.Second, time.Millisecond) + require.Eventually(t, func() bool { return recvNotFound.Load() }, 2*time.Second, time.Millisecond) } func TestGetData(t *testing.T) { From 2cf15ea4e3f898ff096ef96fef298682f24b7ee7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 9 Feb 2022 18:54:13 +0300 Subject: [PATCH 8/9] github: fix publishing job for WSC image Close #2277. --- .github/workflows/publish_to_dockerhub.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index fa634ea630..2496ad7e95 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -128,9 +128,6 @@ jobs: run: make test publish_wsc: # Ensure test job passes before pushing image. - # TODO: currently test_wsc job is failing, so we have `always()` condition. - # After #2269 and #2268 this condition should be removed. - if: ${{ always() }} needs: tests_wsc name: Publish WindowsServerCore-based image to DockerHub runs-on: windows-2022 From 0760e5486b2143e8e3b6d9b6229c13dcff96d465 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 9 Feb 2022 19:40:05 +0300 Subject: [PATCH 9/9] dockerfile: reduce build time for WSC-based image Turns out that caching the golang:windowsservercore-ltsc2022 image between GithubAction workflow runs is a bad idea because `docker load` command still takes too long to load image from cached archive (~9-10min on standard windows runner). And after that runner still needs to build the neo-go image itself. However, standard GA windows runner is supplied with prefetched latest mcr.microsoft.com/windows/servercore:ltsc2022 image, so using it costs almost nothing. Thus, the other approach is implemented: we use standard mcr.microsoft.com/windows/servercore:ltsc2022 image as both build-base and final-base. Then we install all required tools for building neo-go manually on build-base image (these tools are git and go 1.17). Compared to the first approach, the publishing job (included build and publish to DockerHub) takes ~7-8min to finish. --- .github/workflows/publish_to_dockerhub.yml | 3 +++ Dockerfile.wsc | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_to_dockerhub.yml b/.github/workflows/publish_to_dockerhub.yml index 2496ad7e95..724a105991 100644 --- a/.github/workflows/publish_to_dockerhub.yml +++ b/.github/workflows/publish_to_dockerhub.yml @@ -138,6 +138,9 @@ jobs: # Allows to fetch all history for all branches and tags. Need this for proper versioning. fetch-depth: 0 + - name: Show docker images + run: docker images + - name: Build image run: make image-wsc diff --git a/Dockerfile.wsc b/Dockerfile.wsc index 6ed9760abb..0ffcabfeb3 100644 --- a/Dockerfile.wsc +++ b/Dockerfile.wsc @@ -1,5 +1,25 @@ # Builder image -FROM golang:windowsservercore-ltsc2022 as builder +FROM mcr.microsoft.com/windows/servercore:ltsc2022 as builder + +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';", "$ProgressPreference = 'SilentlyContinue';"] + +ENV GIT_VERSION=2.23.0 + +ENV GIT_TAG=v2.23.0.windows.1 + +ENV GIT_DOWNLOAD_URL=https://github.com/git-for-windows/git/releases/download/v2.23.0.windows.1/MinGit-2.23.0-64-bit.zip + +ENV GIT_DOWNLOAD_SHA256=8f65208f92c0b4c3ae4c0cf02d4b5f6791d539cd1a07b2df62b7116467724735 + +RUN Write-Host ('Downloading {0} ...' -f $env:GIT_DOWNLOAD_URL); [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri $env:GIT_DOWNLOAD_URL -OutFile 'git.zip'; Write-Host ('Verifying sha256 ({0}) ...' -f $env:GIT_DOWNLOAD_SHA256); if ((Get-FileHash git.zip -Algorithm sha256).Hash -ne $env:GIT_DOWNLOAD_SHA256) { Write-Host 'FAILED!'; exit 1; }; Write-Host 'Expanding ...'; Expand-Archive -Path git.zip -DestinationPath C:\git\.; Write-Host 'Removing ...'; Remove-Item git.zip -Force; Write-Host 'Updating PATH ...'; $env:PATH = 'C:\git\cmd;C:\git\mingw64\bin;C:\git\usr\bin;' + $env:PATH; [Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine); Write-Host 'Verifying install ("git version") ...'; git version; Write-Host 'Complete.'; + +ENV GOPATH=C:\\go + +RUN $newPath = ('{0}\bin;C:\Program Files\Go\bin;{1}' -f $env:GOPATH, $env:PATH); Write-Host ('Updating PATH: {0}' -f $newPath); [Environment]::SetEnvironmentVariable('PATH', $newPath, [EnvironmentVariableTarget]::Machine); + +ENV GOLANG_VERSION=1.17.6 + +RUN $url = 'https://dl.google.com/go/go1.17.6.windows-amd64.zip'; Write-Host ('Downloading {0} ...' -f $url); [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri $url -OutFile 'go.zip'; $sha256 = '5bf8f87aec7edfc08e6bc845f1c30dba6de32b863f89ae46553ff4bbcc1d4954'; Write-Host ('Verifying sha256 ({0}) ...' -f $sha256); if ((Get-FileHash go.zip -Algorithm sha256).Hash -ne $sha256) { Write-Host 'FAILED!'; exit 1; }; Write-Host 'Expanding ...'; Expand-Archive go.zip -DestinationPath C:\; Write-Host 'Moving ...'; Move-Item -Path C:\go -Destination 'C:\Program Files\Go'; Write-Host 'Removing ...'; Remove-Item go.zip -Force; Write-Host 'Verifying install ("go version") ...'; go version; Write-Host 'Complete.'; COPY . /neo-go