support passing headers and timeout in jrpc calls (#621)

* support passing headers and timeout in jrpc calls

* happy linters...

* also make version have 0 dependencies

* opsa, not checking in negative test

* adding more doc comments

* 1.36.0 prep

* new build image with go 1.18.6. -> new linters -> some changes

.circleci/config.yml

@@ -8,7 +8,7 @@ defaultEnv:
   &defaultEnv
   docker:
     # specify the version
-    - image: docker.io/fortio/fortio.build:v45
+    - image: docker.io/fortio/fortio.build:v46
   working_directory: /go/src/fortio.org/fortio
 
 jobs:

Dockerfile

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v45 as build
+FROM docker.io/fortio/fortio.build:v46 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 ARG MODE=install

Dockerfile.build

@@ -1,5 +1,5 @@
 # Dependencies and linters for build:
-FROM golang:1.18.5
+FROM golang:1.18.6
 # Need gcc for -race test (and some linters though those work with CGO_ENABLED=0)
 RUN apt-get -y update && \
   apt-get --no-install-recommends -y upgrade && \

Dockerfile.echosrv

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v45 as build
+FROM docker.io/fortio/fortio.build:v46 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/echosrv OFFICIAL_BIN=../echosrv.bin

Dockerfile.fcurl

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v45 as build
+FROM docker.io/fortio/fortio.build:v46 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 # fcurl should not need vendor/no dependencies

Makefile

@@ -7,7 +7,7 @@
 IMAGES=echosrv fcurl # plus the combo image / Dockerfile without ext.
 
 DOCKER_PREFIX := docker.io/fortio/fortio
-BUILD_IMAGE_TAG := v45
+BUILD_IMAGE_TAG := v46
 BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
 BUILDX_POSTFIX :=
 ifeq '$(shell echo $(BUILDX_PLATFORMS) | awk -F "," "{print NF-1}")' '0'

README.md

@@ -52,13 +52,13 @@ You can install from source:
 The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets).
 
 ```shell
-curl -L https://github.com/fortio/fortio/releases/download/v1.35.0/fortio-linux_amd64-1.35.0.tgz \
+curl -L https://github.com/fortio/fortio/releases/download/v1.36.0/fortio-linux_amd64-1.36.0.tgz \
  | sudo tar -C / -xvzpf -
 # or the debian package
-wget https://github.com/fortio/fortio/releases/download/v1.35.0/fortio_1.35.0_amd64.deb
-dpkg -i fortio_1.35.0_amd64.deb
+wget https://github.com/fortio/fortio/releases/download/v1.36.0/fortio_1.36.0_amd64.deb
+dpkg -i fortio_1.36.0_amd64.deb
 # or the rpm
-rpm -i https://github.com/fortio/fortio/releases/download/v1.35.0/fortio-1.35.0-1.x86_64.rpm
+rpm -i https://github.com/fortio/fortio/releases/download/v1.36.0/fortio-1.36.0-1.x86_64.rpm
 # and more, see assets in release page
 ```
 
@@ -68,7 +68,7 @@ On a MacOS you can also install Fortio using [Homebrew](https://brew.sh/):
 brew install fortio
 ```
 
-On Windows, download https://github.com/fortio/fortio/releases/download/v1.35.0/fortio_win_1.35.0.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
+On Windows, download https://github.com/fortio/fortio/releases/download/v1.36.0/fortio_win_1.36.0.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
 ```
 fortio.exe server
 ```
@@ -116,7 +116,7 @@ Full list of command line flags (`fortio help`):
 <details>
 <!-- use release/updateFlags.sh to update this section -->
 <pre>
-Φορτίο 1.35.0 usage:
+Φορτίο 1.36.0 usage:
     fortio command [flags] target
 where command is one of: load (load testing), server (starts ui, rest api,
  http-echo, redirect, proxies, tcp-echo and grpc ping servers), tcp-echo (only

Webtest.sh

@@ -125,7 +125,7 @@ fi
 PPROF_URL="$BASE_URL/debug/pprof/heap?debug=1"
 $CURL "$PPROF_URL" | grep -i TotalAlloc # should find this in memory profile
 # creating dummy container to hold a volume for test certs due to remote docker bind mount limitation.
-DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v45 sleep 120)
+DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v46 sleep 120)
 # while we have something with actual curl binary do
 # Test for h2c upgrade (#562)
 docker exec $DOCKERSECVOLNAME /usr/bin/curl -v --http2 -m 10 -d foo42 http://localhost:8080/debug | tee >(cat 1>&2) | grep foo42

fhttp/http_client.go

@@ -32,9 +32,9 @@ import (
 	"time"
 
 	"fortio.org/fortio/fnet"
+	"fortio.org/fortio/jrpc"
 	"fortio.org/fortio/log"
 	"fortio.org/fortio/stats"
-	"fortio.org/fortio/version"
 	"github.com/google/uuid"
 )
 
@@ -151,8 +151,6 @@ func (h *HTTPOptions) URLSchemeCheck() {
 	}
 }
 
-var userAgent = "fortio.org/fortio-" + version.Short()
-
 const (
 	retcodeOffset = len("HTTP/1.X ")
 	// HTTPReqTimeOutDefaultValue is the default timeout value. 3s.
@@ -199,7 +197,7 @@ func (h *HTTPOptions) ResetHeaders() {
 // InitHeaders initialize and/or resets the default headers (ie just User-Agent).
 func (h *HTTPOptions) InitHeaders() {
 	h.ResetHeaders()
-	h.extraHeaders.Add("User-Agent", userAgent)
+	h.extraHeaders.Set(jrpc.UserAgentHeader, jrpc.UserAgent)
 	// No other headers should be added here based on options content as this is called only once
 	// before command line option -H are parsed/set.
 }

fhttp/http_forwarder.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 
 	"fortio.org/fortio/fnet"
+	"fortio.org/fortio/jrpc"
 	"fortio.org/fortio/log"
 )
 
@@ -97,9 +98,9 @@ func MakeSimpleRequest(url string, r *http.Request, copyAllHeaders bool) *http.R
 	// Copy only trace headers or all of them:
 	CopyHeaders(req, r, copyAllHeaders)
 	if copyAllHeaders {
-		req.Header.Add("X-Proxy-Agent", userAgent)
+		req.Header.Add("X-Proxy-Agent", jrpc.UserAgent)
 	} else {
-		req.Header.Add("User-Agent", userAgent)
+		req.Header.Set(jrpc.UserAgentHeader, jrpc.UserAgent)
 	}
 	return req
 }
@@ -117,8 +118,10 @@ func (mcfg *MultiServerConfig) TeeHandler(w http.ResponseWriter, r *http.Request
 	}
 	r.Body.Close()
 	if mcfg.Serial {
+		//nolint:contextcheck // bug I think as we transfer the context - asked in https://github.com/kkHAIKE/contextcheck/issues/3
 		mcfg.TeeSerialHandler(w, r, data)
 	} else {
+		//nolint:contextcheck // bug I think as we transfer the context - asked in https://github.com/kkHAIKE/contextcheck/issues/3
 		mcfg.TeeParallelHandler(w, r, data)
 	}
 }

fhttp/http_server.go

@@ -420,6 +420,7 @@ func FetcherHandler2(w http.ResponseWriter, r *http.Request) {
 	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
 		url = "http://" + url
 	}
+	//nolint:contextcheck // bug(?) we transfer the context from the http request https://github.com/kkHAIKE/contextcheck/issues/3
 	req := MakeSimpleRequest(url, r, fetch2CopiesAllHeader.Get())
 	if req == nil {
 		http.Error(w, "parsing url failed, invalid url", http.StatusBadRequest)

fhttp/http_test.go

@@ -30,6 +30,7 @@ import (
 	"unicode/utf8"
 
 	"fortio.org/fortio/fnet"
+	"fortio.org/fortio/jrpc"
 	"fortio.org/fortio/log"
 	"github.com/google/uuid"
 )
@@ -889,18 +890,18 @@ func TestPayloadForFastClient(t *testing.T) {
 			"application/json",
 			[]byte("{\"test\" : \"test\"}"),
 			fmt.Sprintf("POST / HTTP/1.1\r\nHost: www.google.com\r\nContent-Length: 17\r\nContent-Type: "+
-				"application/json\r\nUser-Agent: %s\r\n\r\n{\"test\" : \"test\"}", userAgent),
+				"application/json\r\nUser-Agent: %s\r\n\r\n{\"test\" : \"test\"}", jrpc.UserAgent),
 		},
 		{
 			"application/xml",
 			[]byte("<test test=\"test\">"),
 			fmt.Sprintf("POST / HTTP/1.1\r\nHost: www.google.com\r\nContent-Length: 18\r\nContent-Type: "+
-				"application/xml\r\nUser-Agent: %s\r\n\r\n<test test=\"test\">", userAgent),
+				"application/xml\r\nUser-Agent: %s\r\n\r\n<test test=\"test\">", jrpc.UserAgent),
 		},
 		{
 			"",
 			nil,
-			fmt.Sprintf("GET / HTTP/1.1\r\nHost: www.google.com\r\nUser-Agent: %s\r\n\r\n", userAgent),
+			fmt.Sprintf("GET / HTTP/1.1\r\nHost: www.google.com\r\nUser-Agent: %s\r\n\r\n", jrpc.UserAgent),
 		},
 	}
 	for _, test := range tests {
@@ -1133,7 +1134,7 @@ func TestDebugHandlerSortedHeaders(t *testing.T) {
 		"Ccc: ccc\n"+
 		"User-Agent: %s\n"+
 		"Zzz: zzz\n\n"+
-		"body:\n\n\n", a.Port, userAgent)
+		"body:\n\n\n", a.Port, jrpc.UserAgent)
 	if body != expected {
 		t.Errorf("Get body: %s not as expected: %s", body, expected)
 	}

jrpc/jrpcClient.go

@@ -16,8 +16,8 @@
 // using generics to serialize/deserialize any type.
 package jrpc // import "fortio.org/fortio/jrpc"
 
-// This package is a true self contained library, doesn't rely on our logger nor other packages in fortio/.
-// Client side and common code.
+// This package is a true self contained library, that doesn't rely on our logger nor other packages
+// in fortio/ outside of version/ (which now also doesn't rely on logger or any other package).
 import (
 	"bytes"
 	"context"
@@ -26,13 +26,27 @@ import (
 	"io"
 	"net/http"
 	"time"
+
+	"fortio.org/fortio/version"
+)
+
+// Client side and common code.
+
+const (
+	UserAgentHeader = "User-Agent"
 )
 
 // Default timeout for Call.
 var timeout = 60 * time.Second
 
+// UserAgent is the User-Agent header used by client calls (also used in fhttp/).
+var UserAgent = "fortio.org/fortio-" + version.Short()
+
 // SetCallTimeout changes the timeout for further Call calls, returns
-// the previous value (default in 60s).
+// the previous value (default in 60s). Value is used when a timeout
+// isn't passed in the options. Note this is not thread safe,
+// use Destination.Timeout for changing values outside of main/single
+// thread.
 func SetCallTimeout(t time.Duration) time.Duration {
 	previous := timeout
 	timeout = t
@@ -50,6 +64,13 @@ type FetchError struct {
 	Bytes []byte
 }
 
+// Destination is the URL and optional additional headers.
+type Destination struct {
+	URL     string
+	Headers *http.Header
+	Timeout time.Duration
+}
+
 func (fe *FetchError) Error() string {
 	return fmt.Sprintf("%s, code %d: %v (raw reply: %s)", fe.Message, fe.Code, fe.Err, DebugSummary(fe.Bytes, 256))
 }
@@ -61,7 +82,7 @@ func (fe *FetchError) Unwrap() error {
 // Call calls the url endpoint, POSTing a serialized as json optional payload
 // (pass nil for a GET http request) and returns the result, deserializing
 // json into type Q. T can be inferred so we declare Response Q first.
-func Call[Q any, T any](url string, payload *T) (*Q, error) {
+func Call[Q any, T any](url *Destination, payload *T) (*Q, error) {
 	var bytes []byte
 	var err error
 	if payload != nil {
@@ -73,23 +94,35 @@ func Call[Q any, T any](url string, payload *T) (*Q, error) {
 	return CallWithPayload[Q](url, bytes)
 }
 
+// CallURL is Call without any options/non default headers, timeout etc and just the URL.
+func CallURL[Q any, T any](url string, payload *T) (*Q, error) {
+	return Call[Q](NewDestination(url), payload)
+}
+
 // CallNoPayload is for an API call without json payload.
-func CallNoPayload[Q any](url string) (*Q, error) {
+func CallNoPayload[Q any](url *Destination) (*Q, error) {
 	return CallWithPayload[Q](url, []byte{})
 }
 
+// CallNoPayloadURL short cut for CallNoPayload with url as a string (default Send()/Destination options).
+func CallNoPayloadURL[Q any](url string) (*Q, error) {
+	return CallWithPayload[Q](NewDestination(url), []byte{})
+}
+
+// Serialize serializes the object as json.
 func Serialize(obj interface{}) ([]byte, error) {
 	return json.Marshal(obj)
 }
 
+// Deserialize deserializes json as a new object of desired type.
 func Deserialize[Q any](bytes []byte) (*Q, error) {
 	var result Q
 	err := json.Unmarshal(bytes, &result)
 	return &result, err // Will return zero object, not nil upon error
 }
 
 // CallWithPayload is for cases where the payload is already serialized (or empty).
-func CallWithPayload[Q any](url string, bytes []byte) (*Q, error) {
+func CallWithPayload[Q any](url *Destination, bytes []byte) (*Q, error) {
 	code, bytes, err := Send(url, bytes) // returns -1 on other errors
 	if err != nil {
 		return nil, err
@@ -110,24 +143,43 @@ func CallWithPayload[Q any](url string, bytes []byte) (*Q, error) {
 	return result, nil
 }
 
+// SetHeaderIfMissing utility function to not overwrite nor append to existing headers.
+func SetHeaderIfMissing(headers http.Header, name, value string) {
+	if headers.Get(name) != "" {
+		return
+	}
+	headers.Set(name, value)
+}
+
 // Send fetches the result from url and sends optional payload as a POST, GET if missing.
 // Returns the http code (if no other error before then, -1 if there are errors),
 // the bytes from the reply and error if any.
-func Send(url string, jsonPayload []byte) (int, []byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+func Send(dest *Destination, jsonPayload []byte) (int, []byte, error) {
+	curTimeout := dest.Timeout
+	if curTimeout == 0 {
+		curTimeout = timeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), curTimeout)
 	defer cancel()
 	var req *http.Request
 	var err error
 	var res []byte
 	if len(jsonPayload) > 0 {
-		req, err = http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonPayload))
-		req.Header.Set("Content-Type", "application/json; charset=utf-8")
+		req, err = http.NewRequestWithContext(ctx, http.MethodPost, dest.URL, bytes.NewReader(jsonPayload))
 	} else {
-		req, err = http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		req, err = http.NewRequestWithContext(ctx, http.MethodGet, dest.URL, nil)
 	}
 	if err != nil {
 		return -1, res, err
 	}
+	if dest.Headers != nil {
+		req.Header = dest.Headers.Clone()
+	}
+	if len(jsonPayload) > 0 {
+		SetHeaderIfMissing(req.Header, "Content-Type", "application/json; charset=utf-8")
+	}
+	SetHeaderIfMissing(req.Header, "Accept", "application/json")
+	SetHeaderIfMissing(req.Header, UserAgentHeader, UserAgent)
 	var resp *http.Response
 	resp, err = http.DefaultClient.Do(req)
 	if err != nil {
@@ -138,8 +190,19 @@ func Send(url string, jsonPayload []byte) (int, []byte, error) {
 	return resp.StatusCode, res, err
 }
 
-// Fetch is Send without a payload.
-func Fetch(url string) (int, []byte, error) {
+// NewDestination returns a Destination object set for the given url
+// (and default/nil replacement headers and default global timeout).
+func NewDestination(url string) *Destination {
+	return &Destination{URL: url}
+}
+
+// FetchURL is Send without a payload and no additional options (default timeout and headers).
+func FetchURL(url string) (int, []byte, error) {
+	return Send(NewDestination(url), []byte{})
+}
+
+// Fetch is Send without a payload (so will be a GET request).
+func Fetch(url *Destination) (int, []byte, error) {
 	return Send(url, []byte{})
 }