new jrpc package, and tests/rework for rest handler (#611)

Adding a new jrpc json rpc client and server library

Use it in tests and in rest handler

Also changed the returned async objects so they can be used and deserialized

Plus improvements

 change list api
 save file async ID and url
 golang 1.18.5
 don't run the build/tests as root

fixes #612, #613

* use generics, cover the other runners, debug the chmod issue

* new json rpc package (#613)

* new jrpc rest package

* shared/standard reply and error objects

* full tests for jrpc, added initial status in grpc (to be changed)

* Update unit-tests to large resource class in config.yml (#614)

* simplification: single base message: use omit empty instead of klunky in go struct inheritance

* change list api to support both all and single status

* update readme

* make release fix, that needs root

* new lint rules

* reuse rapi.Run in ui/uihandler.go. have proper status map result

* update flags from reresolve mr + prep for 1.35.0

* fix abort issue: need to keep the aborter reference as it's cleared from original runner options

* working: send file id back by creating it early

* move data (result) fetching into rapi/ and add tests.

* added fairly complicated started vs not channel/state so abort can work even before Run() is called

* for convenience also translate the id to url in start/stop rest call async replies

* new golang image 1.18.5 which also meant new golanglint-ci which meant quite some pain to change a bunch of stuff for linters

* updated readme, highlight rest in help

* ioutil package is deprecated - replace by io and os equivalents

.circleci/config.yml

@@ -4,18 +4,21 @@
 version: 2
 
 # common setup and steps
-defaultEnv: &defaultEnv
-    docker:
-      # specify the version
-      - image: docker.io/fortio/fortio.build:v42
-    working_directory: /go/src/fortio.org/fortio
+defaultEnv:
+  &defaultEnv
+  docker:
+    # specify the version
+    - image: docker.io/fortio/fortio.build:v45
+  working_directory: /go/src/fortio.org/fortio
 
 jobs:
   unit-tests:
     <<: *defaultEnv
     steps:
       - checkout
       - run: make test
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
+    resource_class: large
   release-tests:
     <<: *defaultEnv
     steps:

.golangci.yml

@@ -127,6 +127,8 @@ issues:
         - gochecknoinits
         - gochecknoglobals
         - forcetypeassert
+        - nosnakecase
+        - noctx
 
     # Exclude lll issues for long lines with go:generate
     - linters:
@@ -141,6 +143,9 @@ issues:
     - linters:
         - godox
       text: "TODO"
+    - linters:
+        - nosnakecase
+      text: "grpc_|_SERVING|O_"
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
   max-issues-per-linter: 0

Dockerfile

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v42 as build
+FROM docker.io/fortio/fortio.build:v45 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 ARG MODE=install

Dockerfile.build

@@ -1,5 +1,5 @@
 # Dependencies and linters for build:
-FROM golang:1.18.4
+FROM golang:1.18.5
 # Need gcc for -race test (and some linters though those work with CGO_ENABLED=0)
 RUN apt-get -y update && \
   apt-get --no-install-recommends -y upgrade && \
@@ -22,5 +22,8 @@ RUN set -x; if [ x"$(dpkg --print-architecture)" != x"s390x" ]; then \
     fi
 
 WORKDIR /build
-VOLUME /build
 COPY .golangci.yml .
+VOLUME /build
+RUN useradd -m build -d /build
+RUN chown -R build:build /build
+USER build

Dockerfile.echosrv

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v42 as build
+FROM docker.io/fortio/fortio.build:v45 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/echosrv OFFICIAL_BIN=../echosrv.bin

Dockerfile.fcurl

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v42 as build
+FROM docker.io/fortio/fortio.build:v45 as build
 WORKDIR /go/src/fortio.org
 COPY . fortio
 # fcurl should not need vendor/no dependencies

Makefile

@@ -7,7 +7,7 @@
 IMAGES=echosrv fcurl # plus the combo image / Dockerfile without ext.
 
 DOCKER_PREFIX := docker.io/fortio/fortio
-BUILD_IMAGE_TAG := v42
+BUILD_IMAGE_TAG := v45
 BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
 BUILDX_POSTFIX :=
 ifeq '$(shell echo $(BUILDX_PLATFORMS) | awk -F "," "{print NF-1}")' '0'
@@ -57,6 +57,7 @@ test: dependencies
 # DEBUG_LINTERS="--debug"
 
 local-lint:
+	golangci-lint version
 	golangci-lint $(DEBUG_LINTERS) run $(LINT_PACKAGES)
 
 # Lint everything by default but ok to "make lint LINT_PACKAGES=./fhttp"
@@ -71,6 +72,9 @@ docker-test:
 		"cd /go/src/fortio.org/fortio \
 		&& time make test"
 
+shell:
+	docker run -ti -v $(CURDIR):/go/src/fortio.org/fortio $(BUILD_IMAGE)
+
 # This really also tests the release process and build on windows,mac,linux
 # and the docker images, not just "web" (ui) stuff that it also exercises.
 release-test:

README.md

@@ -28,7 +28,8 @@ Fortio is quite mature and very stable with no known major bugs (lots of possibl
 and when bugs are found they are fixed quickly, so after 1 year of development and 42 incremental releases, we reached 1.0 in June 2018.
 
 Fortio components can be used a library even for unrelated projects, for instance the `log`, `stats`, or `fhttp` utilities both client and server.
-As well as the newly integrated [Dynamic Flags](dflag/) support (greatly inspired/imported initially from https://github.com/mwitkow/go-flagz)
+As well as the newly integrated [Dynamic Flags](dflag/) support (greatly inspired/imported initially from https://github.com/mwitkow/go-flagz but recently reimplemented using Go generics).
+Even more recent is the new `jrpc` JSON Remote Procedure Calls library package ([docs](https://pkg.go.dev/fortio.org/fortio/jrpc)).
 
 If you want to connect to fortio using https and fortio to provide real TLS certificates, or to multiplex grpc and regular http behind a single port, check out [Fortio Proxy](https://github.com/fortio/proxy#fortio-proxy)
 
@@ -51,13 +52,13 @@ You can install from source:
 The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets).
 
 ```shell
-curl -L https://github.com/fortio/fortio/releases/download/v1.34.1/fortio-linux_amd64-1.34.1.tgz \
+curl -L https://github.com/fortio/fortio/releases/download/v1.35.0/fortio-linux_amd64-1.35.0.tgz \
  | sudo tar -C / -xvzpf -
 # or the debian package
-wget https://github.com/fortio/fortio/releases/download/v1.34.1/fortio_1.34.1_amd64.deb
-dpkg -i fortio_1.34.1_amd64.deb
+wget https://github.com/fortio/fortio/releases/download/v1.35.0/fortio_1.35.0_amd64.deb
+dpkg -i fortio_1.35.0_amd64.deb
 # or the rpm
-rpm -i https://github.com/fortio/fortio/releases/download/v1.34.1/fortio-1.34.1-1.x86_64.rpm
+rpm -i https://github.com/fortio/fortio/releases/download/v1.35.0/fortio-1.35.0-1.x86_64.rpm
 # and more, see assets in release page
 ```
 
@@ -67,7 +68,7 @@ On a MacOS you can also install Fortio using [Homebrew](https://brew.sh/):
 brew install fortio
 ```
 
-On Windows, download https://github.com/fortio/fortio/releases/download/v1.34.1/fortio_win_1.34.1.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
+On Windows, download https://github.com/fortio/fortio/releases/download/v1.35.0/fortio_win_1.35.0.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
 ```
 fortio.exe server
 ```
@@ -115,13 +116,14 @@ Full list of command line flags (`fortio help`):
 <details>
 <!-- use release/updateFlags.sh to update this section -->
 <pre>
-Φορτίο 1.34.1 usage:
-where command is one of: load (load testing), server (starts ui, http-echo,
- redirect, proxies, tcp-echo and grpc ping servers), tcp-echo (only the tcp-echo
- server), report (report only UI server), redirect (only the redirect server),
- proxies (only the -M and -P configured proxies), grpcping (grpc client),
- or curl (single URL debug), or nc (single tcp or udp:// connection),
- or version (prints the version).
+Φορτίο 1.35.0 usage:
+    fortio command [flags] target
+where command is one of: load (load testing), server (starts ui, rest api,
+ http-echo, redirect, proxies, tcp-echo and grpc ping servers), tcp-echo (only
+ the tcp-echo server), report (report only UI server), redirect (only the
+ redirect server), proxies (only the -M and -P configured proxies), grpcping
+ (grpc client), or curl (single URL debug), or nc (single tcp or udp://
+ connection), or version (prints the full version and build details).
 where target is a url (http load tests) or host:port (grpc health test).
 flags are:
   -H header
@@ -254,6 +256,9 @@ true)
 is to use duration (-t). Default is 1 when used as grpc ping count.
   -nc-dont-stop-on-eof
         in netcat (nc) mode, don't abort as soon as remote side closes
+  -no-reresolve
+        Keep the initial DNS resolution and don't re-resolve when making new
+connections (because of error or reuse limit reached)
   -nocatchup
         set to exact fixed qps and prevent fortio from trying to catchup when
 the target fails to keep up temporarily
@@ -370,7 +375,8 @@ You can set a default value for all these by passing `-echo-server-default-param
 
 * API to trigger and cancel runs from the running server (like the form ui but more directly and with `async=on` option)
   * `/fortio/rest/run` starts a run; the arguments are either from the command line or from POSTed JSON; `jsonPath` can be provided to look for in a subset of the json object, for instance `jsonPath=metadata` allows to use the flagger webhook meta data for fortio run parameters (see [Remote Triggered load test section below](#remote-triggered-load-test-server-mode-rest-api)).
-  * `/fortio/rest/stop` stops all current run or by run id.
+  * `/fortio/rest/stop` stops all current run or by run id (passing `runid=` query argument).
+  * `/fortio/rest/status` lists the current runs (or the options of a single one if `runid` is passed).
 
 The `report` mode is a readonly subset of the above directly on `/`.
 
@@ -388,11 +394,12 @@ Fortio X.Y.Z grpc 'ping' server listening on tcp [::]:8079
 Fortio X.Y.Z https redirector server listening on tcp [::]:8081
 Fortio X.Y.Z http-echo server listening on tcp [::]:8080
 Data directory is /Users/ldemailly/dev/fortio
-UI started - visit:
-http://localhost:8080/fortio/
-(or any host/ip reachable on this server)
-14:11:05 I fortio_main.go:285> Note: not using dynamic flag watching (use -config to set watch directory)
-14:11:05 I fortio_main.go:293> All fortio X.Y.Z unknown goM.m.p servers started!
+REST API on /fortio/rest/run, /fortio/rest/status, /fortio/rest/stop
+	 UI started - visit:
+		http://localhost:8080/fortio/
+	 (or any host/ip reachable on this server)
+I fortio_main.go:285> Note: not using dynamic flag watching (use -config to set watch directory)
+I fortio_main.go:293> All fortio X.Y.Z  goM.m.p arm64 darwin servers started!
 ```
 
 ### Sample of the graphing UI

Webtest.sh

@@ -125,7 +125,7 @@ fi
 PPROF_URL="$BASE_URL/debug/pprof/heap?debug=1"
 $CURL "$PPROF_URL" | grep -i TotalAlloc # should find this in memory profile
 # creating dummy container to hold a volume for test certs due to remote docker bind mount limitation.
-DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v42 sleep 120)
+DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v45 sleep 120)
 # while we have something with actual curl binary do
 # Test for h2c upgrade (#562)
 docker exec $DOCKERSECVOLNAME /usr/bin/curl -v --http2 -m 10 -d foo42 http://localhost:8080/debug | tee >(cat 1>&2) | grep foo42

dflag/configmap/updater.go

@@ -9,7 +9,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path"
 	"strings"
 
@@ -110,7 +110,7 @@ func (u *Updater) Stop() error {
 }
 
 func (u *Updater) readAll(dynamicOnly bool) error {
-	files, err := ioutil.ReadDir(u.dirPath)
+	files, err := os.ReadDir(u.dirPath)
 	if err != nil {
 		return fmt.Errorf("dflag: updater initialization: %w", err)
 	}
@@ -145,7 +145,7 @@ func (u *Updater) readFlagFile(fullPath string, dynamicOnly bool) error {
 	if dynamicOnly && !dflag.IsFlagDynamic(flag) {
 		return errFlagNotDynamic
 	}
-	content, err := ioutil.ReadFile(fullPath)
+	content, err := os.ReadFile(fullPath)
 	if err != nil {
 		return err
 	}
@@ -161,7 +161,7 @@ func (u *Updater) watchForUpdates() {
 		select {
 		case event := <-u.watcher.Events:
 			log.LogVf("ConfigMap got fsnotify %v ", event)
-			if event.Name == u.dirPath || event.Name == path.Join(u.dirPath, k8sDataSymlink) { // nolint: nestif
+			if event.Name == u.dirPath || event.Name == path.Join(u.dirPath, k8sDataSymlink) { //nolint:nestif
 				// case of the whole directory being re-symlinked
 				switch event.Op {
 				case fsnotify.Create:

dflag/configmap/updater_test.go

@@ -5,7 +5,6 @@ package configmap_test
 
 import (
 	"flag"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -43,7 +42,7 @@ type updaterTestSuite struct {
 
 func (s *updaterTestSuite) SetupTest() {
 	var err error
-	s.tempDir, err = ioutil.TempDir("/tmp", "updater_test")
+	s.tempDir, err = os.MkdirTemp("/tmp", "updater_test")
 	require.NoError(s.T(), err, "failed creating temp directory for testing")
 	s.copyTestDataToDir()
 	s.linkDataDirTo(firstGoodDir)

dflag/dynint64_test.go

@@ -63,7 +63,7 @@ func Benchmark_Int64_Dyn_Get(b *testing.B) {
 	set.Set("some_int_1", "77007700")
 	for i := 0; i < b.N; i++ {
 		x := value.Get()
-		x++ // nolint: ineffassign
+		x++ //nolint:ineffassign
 	}
 }
 
@@ -73,6 +73,6 @@ func Benchmark_Int64_Normal_Get(b *testing.B) {
 	set.Set("some_int_1", "77007700")
 	for i := 0; i < b.N; i++ {
 		x := *valPtr
-		x++ // nolint: ineffassign
+		x++ //nolint:ineffassign
 	}
 }

dflag/dynstring_test.go

@@ -78,7 +78,7 @@ func Benchmark_String_Dyn_Get(b *testing.B) {
 	set.Set("some_string_1", "else")
 	for i := 0; i < b.N; i++ {
 		x := value.Get()
-		x += "foo" // nolint: ineffassign
+		x += "foo" //nolint:ineffassign
 	}
 }
 
@@ -88,6 +88,6 @@ func Benchmark_String_Normal_get(b *testing.B) {
 	set.Set("some_string_1", "else")
 	for i := 0; i < b.N; i++ {
 		x := *valPtr
-		x += "foo" // nolint: ineffassign
+		x += "foo" //nolint:ineffassign
 	}
 }

dflag/endpoint/endpoint.go

@@ -113,7 +113,7 @@ func requestIsBrowser(req *http.Request) bool {
 	return strings.Contains(req.Header.Get("Accept"), "html")
 }
 
-// nolint: lll
+//nolint:lll
 var dflagListTemplate = template.Must(template.New("dflag_list").Parse(
 	`
 <html><head>

dflag/endpoint/endpoint_test.go

@@ -47,43 +47,43 @@ func (s *endpointTestSuite) SetupTest() {
 }
 
 func (s *endpointTestSuite) TestReturnsAll() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_static_string", "some_static_float", "some_dyn_stringslice", "some_dyn_json"}, list)
 }
 
 func (s *endpointTestSuite) TestReturnsOnlyChanged() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag?only_changed=true", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag?only_changed=true", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_static_string", "some_dyn_stringslice"}, list)
 }
 
 func (s *endpointTestSuite) TestReturnsOnlyStatic() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag?type=static", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag?type=static", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_static_string", "some_static_float"}, list)
 }
 
 func (s *endpointTestSuite) TestReturnsOnlyDynamic() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag?type=dynamic", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag?type=dynamic", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_dyn_stringslice", "some_dyn_json"}, list)
 }
 
 func (s *endpointTestSuite) TestReturnsOnlyDynamicAndChanged() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag?type=dynamic&only_changed=true", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag?type=dynamic&only_changed=true", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_dyn_stringslice"}, list)
 }
 
 func (s *endpointTestSuite) TestReturnsOnlyStaticAndChanged() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag?type=static&only_changed=true", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag?type=static&only_changed=true", nil)
 	list := s.processFlagSetJSONResponse(req)
 	s.assertListContainsOnly([]string{"some_static_string"}, list)
 }
 
 func (s *endpointTestSuite) TestCorrectlyRepresentsResources() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag", nil)
 	list := s.processFlagSetJSONResponse(req)
 
 	assert.Equal(s.T(),
@@ -113,7 +113,7 @@ func (s *endpointTestSuite) TestCorrectlyRepresentsResources() {
 }
 
 func (s *endpointTestSuite) TestServesHTML() {
-	req, _ := http.NewRequestWithContext(context.Background(), "GET", "/debug/dflag", nil)
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/debug/dflag", nil)
 	req.Header.Add("Accept", "application/xhtml+xml")
 	resp := httptest.NewRecorder()
 	s.endpoint.ListFlags(resp, req)

dflag/fileread_flag.go

@@ -6,7 +6,7 @@ package dflag
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"os"
 )
 
 // ReadFileFlags parses the flagset to discover all "fileread" flags and evaluates them.
@@ -60,7 +60,7 @@ func (f *FileReadValue) readFile() error {
 	if f.filePath == "" {
 		return nil
 	}
-	data, err := ioutil.ReadFile(f.filePath)
+	data, err := os.ReadFile(f.filePath)
 	if err != nil {
 		return err
 	}

dflag/test_utils.go

@@ -136,7 +136,7 @@ func (d *Testify) Run(t *testing.T, suite hasT) {
 	}
 	for i := 0; i < methodFinder.NumMethod(); i++ {
 		method := methodFinder.Method(i)
-		// nolint: staticcheck // consider fixing later for perf but this is just to run a few tests.
+		//nolint:staticcheck // consider fixing later for perf but this is just to run a few tests.
 		if ok, _ := regexp.MatchString("^Test", method.Name); !ok {
 			continue
 		}