From 98e23a35292a0729ca961ea66890e02f5ba1ed0a Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Tue, 11 May 2021 17:46:39 +0800 Subject: [PATCH 1/5] format allocate_test.go --- allocate_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/allocate_test.go b/allocate_test.go index 892826a0..fb50a451 100644 --- a/allocate_test.go +++ b/allocate_test.go @@ -356,7 +356,7 @@ func TestModifyCmdFunc(t *testing.T) { allocCtx, cancel := NewExecAllocator(context.Background(), append([]ExecAllocatorOption{ ModifyCmdFunc(func(cmd *exec.Cmd) { - cmd.Env = append(cmd.Env, "TZ=" + tz) + cmd.Env = append(cmd.Env, "TZ="+tz) }), }, allocOpts...)...) defer cancel() @@ -375,4 +375,3 @@ func TestModifyCmdFunc(t *testing.T) { t.Fatalf("got %s, want %s", ret, tz) } } - From 4fe56159912020667587f887617f15a859fd89c2 Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Tue, 11 May 2021 17:51:39 +0800 Subject: [PATCH 2/5] make NewRemoteAllocator accept url without "devtools/browser/..." A full websocket debugger URL is something like this: ws://127.0.0.1:9222/devtools/browser/120e6fb7-0094-47a2-8b4c-51741797f280 And the ID part is not static, which requires extra work to get the correct URL for NewRemoteAllocator. This change allows the client to just pass the base URL to NewRemoteAllocator. It will send a request to "/json/version" to get the full websocket debugger URL. After the change, the URL with the following formats are accepted: * ws://127.0.0.1:9222/ * http://127.0.0.1:9222/ But "ws://127.0.0.1:9222/devtools/browser/" are not accepted. Because it contains "/devtools/browser/" and will be considered as a valid websocket debugger URL. --- allocate.go | 14 ++++++++++++++ allocate_test.go | 31 ++++++++++++++++++++++++++++++- util.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/allocate.go b/allocate.go index 1ae5254f..1dbd9b19 100644 --- a/allocate.go +++ b/allocate.go @@ -478,7 +478,21 @@ func CombinedOutput(w io.Writer) ExecAllocatorOption { // NewRemoteAllocator creates a new context set up with a RemoteAllocator, // suitable for use with NewContext. The url should point to the browser's // websocket address, such as "ws://127.0.0.1:$PORT/devtools/browser/...". +// If the url does not contain "/devtools/browser/", it will try to detect +// the correct one by sending a request to "http://$HOST:$PORT/json/version". +// +// The url with the following formats are accepted: +// * ws://127.0.0.1:9222/ +// * http://127.0.0.1:9222/ +// +// But "ws://127.0.0.1:9222/devtools/browser/" are not accepted. +// Because it contains "/devtools/browser/" and will be considered +// as a valid websocket debugger URL. func NewRemoteAllocator(parent context.Context, url string) (context.Context, context.CancelFunc) { + url, err := detectURL(url) + if err != nil { + panic(fmt.Sprintf("failed to detect the websocket debugger url: %v", err)) + } ctx, cancel := context.WithCancel(parent) c := &Context{Allocator: &RemoteAllocator{ wsURL: url, diff --git a/allocate_test.go b/allocate_test.go index fb50a451..c7dc5c6a 100644 --- a/allocate_test.go +++ b/allocate_test.go @@ -122,6 +122,35 @@ func TestSkipNewContext(t *testing.T) { func TestRemoteAllocator(t *testing.T) { t.Parallel() + tests := []struct { + name string + modifyURL func(wsURL string) string + }{ + { + name: "original wsURL", + modifyURL: func(wsURL string) string { return wsURL }, + }, + { + name: "detect from ws", + modifyURL: func(wsURL string) string { + return wsURL[0:strings.Index(wsURL, "devtools")] + }, + }, + { + name: "detect from http", + modifyURL: func(wsURL string) string { + return "http" + wsURL[2:strings.Index(wsURL, "devtools")] + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testRemoteAllocator(t, tt.modifyURL) + }) + } +} + +func testRemoteAllocator(t *testing.T, modifyURL func(wsURL string) string) { tempDir, err := ioutil.TempDir("", "chromedp-runner") if err != nil { t.Fatal(err) @@ -156,7 +185,7 @@ func TestRemoteAllocator(t *testing.T) { if err != nil { t.Fatal(err) } - allocCtx, allocCancel := NewRemoteAllocator(context.Background(), wsURL) + allocCtx, allocCancel := NewRemoteAllocator(context.Background(), modifyURL(wsURL)) defer allocCancel() taskCtx, taskCancel := NewContext(allocCtx, diff --git a/util.go b/util.go index 1c2e1bf1..13093092 100644 --- a/util.go +++ b/util.go @@ -1,8 +1,11 @@ package chromedp import ( + "encoding/json" "net" + "net/http" "net/url" + "strings" "github.com/chromedp/cdproto" "github.com/chromedp/cdproto/cdp" @@ -29,6 +32,47 @@ func forceIP(urlstr string) string { return u.String() } +// detectURL detects the websocket debugger URL if the provided URL is not a +// valid websocket debugger URL. +// +// A valid websocket debugger URL is something like: +// ws://127.0.0.1:9222/devtools/browser/... +// The original URL with the following formats are accepted: +// * ws://127.0.0.1:9222/ +// * http://127.0.0.1:9222/ +func detectURL(urlstr string) (string, error) { + if strings.Contains(urlstr, "/devtools/browser/") { + return urlstr, nil + } + + // replace the scheme and path to construct the URL like: + // http://127.0.0.1:9222/json/version + u, err := url.Parse(urlstr) + if err != nil { + return "", err + } + u.Scheme = "http" + u.Path = "/json/version" + + // to get "webSocketDebuggerUrl" in the response + resp, err := http.Get(u.String()) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + // the browser will construct the debugger URL using the "host" header of the /json/version request. + // for example, run headless-shell in a container: docker run -d -p 9000:9222 chromedp/headless-shell:latest + // then: curl http://127.0.0.1:9000/json/version + // and the debugger URL will be something like: ws://127.0.0.1:9000/devtools/browser/... + wsURL := result["webSocketDebuggerUrl"].(string) + return wsURL, nil +} + func runListeners(list []cancelableListener, ev interface{}) []cancelableListener { for i := 0; i < len(list); { listener := list[i] From 3673c0cb5023f1fd834d2c8922acd9614623ce8c Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 12 May 2021 11:04:41 +0800 Subject: [PATCH 3/5] use original url instead of panic, to align with forceIP --- allocate.go | 6 +----- util.go | 12 ++++++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/allocate.go b/allocate.go index 1dbd9b19..2c9d7365 100644 --- a/allocate.go +++ b/allocate.go @@ -489,13 +489,9 @@ func CombinedOutput(w io.Writer) ExecAllocatorOption { // Because it contains "/devtools/browser/" and will be considered // as a valid websocket debugger URL. func NewRemoteAllocator(parent context.Context, url string) (context.Context, context.CancelFunc) { - url, err := detectURL(url) - if err != nil { - panic(fmt.Sprintf("failed to detect the websocket debugger url: %v", err)) - } ctx, cancel := context.WithCancel(parent) c := &Context{Allocator: &RemoteAllocator{ - wsURL: url, + wsURL: detectURL(url), }} ctx = context.WithValue(ctx, contextKey{}, c) return ctx, cancel diff --git a/util.go b/util.go index 13093092..74cbf1fd 100644 --- a/util.go +++ b/util.go @@ -40,16 +40,16 @@ func forceIP(urlstr string) string { // The original URL with the following formats are accepted: // * ws://127.0.0.1:9222/ // * http://127.0.0.1:9222/ -func detectURL(urlstr string) (string, error) { +func detectURL(urlstr string) string { if strings.Contains(urlstr, "/devtools/browser/") { - return urlstr, nil + return urlstr } // replace the scheme and path to construct the URL like: // http://127.0.0.1:9222/json/version u, err := url.Parse(urlstr) if err != nil { - return "", err + return urlstr } u.Scheme = "http" u.Path = "/json/version" @@ -57,20 +57,20 @@ func detectURL(urlstr string) (string, error) { // to get "webSocketDebuggerUrl" in the response resp, err := http.Get(u.String()) if err != nil { - return "", err + return urlstr } defer resp.Body.Close() var result map[string]interface{} if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return "", err + return urlstr } // the browser will construct the debugger URL using the "host" header of the /json/version request. // for example, run headless-shell in a container: docker run -d -p 9000:9222 chromedp/headless-shell:latest // then: curl http://127.0.0.1:9000/json/version // and the debugger URL will be something like: ws://127.0.0.1:9000/devtools/browser/... wsURL := result["webSocketDebuggerUrl"].(string) - return wsURL, nil + return wsURL } func runListeners(list []cancelableListener, ev interface{}) []cancelableListener { From 83892fd4b8187348df81c410aa3044c3aa5c380a Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 12 May 2021 11:22:27 +0800 Subject: [PATCH 4/5] support host/domain name in the original url --- allocate_test.go | 22 ++++++++++++++++++++++ util.go | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/allocate_test.go b/allocate_test.go index c7dc5c6a..e482a9c6 100644 --- a/allocate_test.go +++ b/allocate_test.go @@ -5,8 +5,10 @@ import ( "context" "fmt" "io/ioutil" + "net" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "strings" @@ -142,6 +144,26 @@ func TestRemoteAllocator(t *testing.T) { return "http" + wsURL[2:strings.Index(wsURL, "devtools")] }, }, + { + name: "hostname", + modifyURL: func(wsURL string) string { + h, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(wsURL) + if err != nil { + t.Fatal(err) + } + _, post, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatal(err) + } + u.Host = net.JoinHostPort(h, post) + u.Path = "/" + return u.String() + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/util.go b/util.go index 74cbf1fd..2e56aca9 100644 --- a/util.go +++ b/util.go @@ -55,7 +55,7 @@ func detectURL(urlstr string) string { u.Path = "/json/version" // to get "webSocketDebuggerUrl" in the response - resp, err := http.Get(u.String()) + resp, err := http.Get(forceIP(u.String())) if err != nil { return urlstr } From 33fef110aefd44190f56380f259b2f176643666f Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 12 May 2021 12:18:34 +0800 Subject: [PATCH 5/5] make the browser created for testing remote allocator accept all connections --- allocate_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/allocate_test.go b/allocate_test.go index e482a9c6..f2132b1d 100644 --- a/allocate_test.go +++ b/allocate_test.go @@ -191,6 +191,7 @@ func testRemoteAllocator(t *testing.T, modifyURL func(wsURL string) string) { // TODO: perhaps deduplicate this code with ExecAllocator "--user-data-dir="+tempDir, + "--remote-debugging-address=0.0.0.0", "--remote-debugging-port=0", "about:blank", )