Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

adds support to have nowUTCToken #866

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

adds support to have nowUTCToken #866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d47b55
ci: upgrade go version since tests rely on go1.19 for sync/atomic Bool
Dec 2, 2023
9ea9c3e
feat: add nowUTCToken to set all instances to the same now in UTC
Dec 2, 2023
483b16b
ci: fix lint issue
Dec 2, 2023
486d46b
feat; added nowUTCToken interopolation to fastclient
Dec 2, 2023
3fc169e
refactor: address replaceall comment
Dec 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Expand Up @@ -51,7 +51,7 @@ TEST_TIMEOUT:=90s

# Local test
test: dependencies
go test -timeout $(TEST_TIMEOUT) -race $(PACKAGES)
go test -timeout $(TEST_TIMEOUT) -race -count=1 $(PACKAGES)

# To debug strange linter errors, uncomment
# DEBUG_LINTERS="--debug"
Expand Down
23 changes: 22 additions & 1 deletion fhttp/http_client.go
View file
Open in desktop
Expand Up @@ -60,9 +60,12 @@ type Fetcher interface {
}

const (
uuidToken = "{uuid}"
uuidToken = "{uuid}"
nowUTCToken = "{nowUTC}"
)

var nowFn = time.Now

var (
// BufferSizeKb size of the buffer (max data) for optimized client in kilobytes defaults to 128k.
BufferSizeKb = 128
Expand Down Expand Up @@ -435,6 +438,7 @@ type Client struct {
pathContainsUUID bool // if url contains the "{uuid}" pattern (lowercase)
rawQueryContainsUUID bool // if any query params contains the "{uuid}" pattern (lowercase)
bodyContainsUUID bool // if body contains the "{uuid}" pattern (lowercase)
bodyContainsNOWUTC bool // if body contains the "{nowUTC}" pattern (lowercase)
logErrors bool
id int
runID int64
Expand Down Expand Up @@ -513,12 +517,22 @@ func (c *Client) StreamFetch(ctx context.Context) (int, int64, uint) {
for strings.Contains(body, uuidToken) {
body = strings.Replace(body, uuidToken, generateUUID(), 1)
}
c.body = []byte(body)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this needed but wasn't for {uuid}

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when there's a body that has both payloads it would overwrite the req.Body without the {uuid} interpolation. so I made sure that we updated it before processing down the chain

bodyBytes := []byte(body)
req.ContentLength = int64(len(bodyBytes))
req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
} else if len(c.body) > 0 {
req.Body = io.NopCloser(bytes.NewReader(c.body))
}

if c.bodyContainsNOWUTC {
nowUTC := nowFn().UTC()
body := strings.ReplaceAll(string(c.body), nowUTCToken, nowUTC.String())
bodyBytes := []byte(body)
req.ContentLength = int64(len(bodyBytes))
req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
}

resp, err := c.client.Do(req)
if err != nil {
log.S(log.Error, "Unable to send request",
Expand Down Expand Up @@ -594,6 +608,7 @@ func NewStdClient(o *HTTPOptions) (*Client, error) {
rawQueryContainsUUID: strings.Contains(req.URL.RawQuery, uuidToken),
body: o.Payload,
bodyContainsUUID: strings.Contains(string(o.Payload), uuidToken),
bodyContainsNOWUTC: strings.Contains(string(o.Payload), nowUTCToken),
req: req,
client: &http.Client{
Timeout: o.HTTPReqTimeOut,
Expand Down Expand Up @@ -801,6 +816,12 @@ func NewFastClient(o *HTTPOptions) (Fetcher, error) { //nolint:funlen
if len(uuidStrings) > 0 {
o.Payload = []byte(payload)
}

nowUTC := nowFn().UTC()
if strings.Contains(payload, nowUTCToken) {
o.Payload = []byte(strings.ReplaceAll(payload, nowUTCToken, nowUTC.String()))
}

// Parse the url, extract components.
url, err := url.Parse(urlString)
if err != nil {
Expand Down
188 changes: 188 additions & 0 deletions fhttp/http_test.go
View file
Open in desktop
Expand Up @@ -1041,6 +1041,100 @@ func TestUUIDPayloadFastClient(t *testing.T) {
}
}

func TestNOWUTCNowPayloadFastClient(t *testing.T) {
// Given
m, a := DynamicHTTPServer(false)
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// set the request to the response for assertion in this test
reqBytes, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("error setting response body: %s", err.Error())
}
w.WriteHeader(http.StatusOK)
w.Write(reqBytes)
})
url := fmt.Sprintf("http://localhost:%d/", a.Port)
o := HTTPOptions{
URL: url,
DisableFastClient: false,
Payload: []byte("[\"{nowUTC}\", \"{nowUTC}\"]"),
}
// make deterministic time
nowTime := time.Now()
nowFn = func() time.Time {
return nowTime
}
client, _ := NewClient(&o)

// When
code, data, header := client.Fetch(context.Background())
t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)

// THen
if code != 200 {
t.Errorf("Got %d instead of 200", code)
}

expectedData := fmt.Sprintf("[%q, %q]", nowTime.UTC(), nowTime.UTC())
if !strings.Contains(string(data), expectedData) {
t.Errorf("Got %s instead of %s", string(data), expectedData)
}
}

func TestUUIDAndUTCNowPayloadFastClient(t *testing.T) {
// Given
m, a := DynamicHTTPServer(false)
interpolatedPayload := []byte{}
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// set the request to the response for assertion in this test
reqBytes, err := io.ReadAll(r.Body)
t.Log(string(reqBytes))
if err != nil {
t.Fatalf("error setting response body: %s", err.Error())
}
w.WriteHeader(http.StatusOK)
interpolatedPayload = reqBytes
})
url := fmt.Sprintf("http://localhost:%d/", a.Port)
o := HTTPOptions{
URL: url,
DisableFastClient: false,
Payload: []byte("[\"{uuid}\", \"{nowUTC}\"]"),
}
// make deterministic time
nowTime := time.Now()
nowFn = func() time.Time {
return nowTime
}

client, _ := NewClient(&o)

// When
code, data, header := client.Fetch(context.Background())
t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)

// Then
if code != 200 {
t.Errorf("Got %d instead of 200", code)
}
var sliceString []string
err := json.NewDecoder(bytes.NewReader(interpolatedPayload)).Decode(&sliceString)
if err != nil {
t.Fatalf("problem deserializing data: %s", err.Error())
}
if len(sliceString) != 2 {
t.Fatalf("Got len %d want 2", len(sliceString))
}

if _, err := uuid.Parse(sliceString[0]); err != nil {
t.Fatalf("%s is not a valid uuid", sliceString[0])
}

if sliceString[1] != nowTime.UTC().String() {
t.Fatalf("Got %s want %s", sliceString[1], nowTime.UTC().String())
}
}

func TestQueryUUIDFastClient(t *testing.T) {
m, a := DynamicHTTPServer(false)
m.HandleFunc("/", ValidateUUIDQueryParam)
Expand Down Expand Up @@ -1127,6 +1221,100 @@ func TestUUIDPayloadClient(t *testing.T) {
}
}

func TestNOWUTCPayloadClient(t *testing.T) {
// Given
m, a := DynamicHTTPServer(false)
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// set the request to the response for assertion in this test
reqBytes, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("error setting response body: %s", err.Error())
}
w.WriteHeader(http.StatusOK)
w.Write(reqBytes)
})
url := fmt.Sprintf("http://localhost:%d/", a.Port)
o := HTTPOptions{
URL: url,
DisableFastClient: true,
Payload: []byte("[\"{nowUTC}\", \"{nowUTC}\"]"),
}
// make deterministic time
nowTime := time.Now()
nowFn = func() time.Time {
return nowTime
}
client, _ := NewClient(&o)

// When
code, data, header := client.Fetch(context.Background())
t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)

// Then
if code != 200 {
t.Errorf("Got %d instead of 200", code)
}

expectedDataString := fmt.Sprintf("[%q, %q]", nowTime.UTC(), nowTime.UTC())
if string(data) != expectedDataString {
t.Errorf("Got %s instead of %s", string(data), expectedDataString)
}
}

func TestUUIDAndNOWUTCPayloadClient(t *testing.T) {
// Given
m, a := DynamicHTTPServer(false)
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// set the request to the response for assertion in this test
reqBytes, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("error setting response body: %s", err.Error())
}
w.WriteHeader(http.StatusOK)
w.Write(reqBytes)
})
url := fmt.Sprintf("http://localhost:%d/", a.Port)
o := HTTPOptions{
URL: url,
DisableFastClient: true,
Payload: []byte("[\"{uuid}\", \"{nowUTC}\"]"),
}
// make deterministic time
nowTime := time.Now()
nowFn = func() time.Time {
return nowTime
}

client, _ := NewClient(&o)

// When
code, data, header := client.Fetch(context.Background())
t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)

// Then
if code != 200 {
t.Errorf("Got %d instead of 200", code)
}

// layout := "2006-01-02 15:04:05"
var sliceString []string
err := json.NewDecoder(bytes.NewReader(data)).Decode(&sliceString)
if err != nil {
t.Fatalf("problem deserializing data: %s", err.Error())
}
if len(sliceString) != 2 {
t.Fatalf("Got len %d want 2", len(sliceString))
}

if _, err := uuid.Parse(sliceString[0]); err != nil {
t.Fatalf("%s is not a valid uuid", sliceString[0])
}

if sliceString[1] != nowTime.UTC().String() {
t.Fatalf("Got %s want %s", sliceString[1], nowTime.UTC().String())
}
}

func TestQueryUUIDClient(t *testing.T) {
m, a := DynamicHTTPServer(false)
m.HandleFunc("/", ValidateUUIDQueryParam)
Expand Down
13 changes: 7 additions & 6 deletions go.mod
View file
Open in desktop
@@ -1,6 +1,6 @@
module fortio.org/fortio

go 1.18
go 1.19

require (
fortio.org/assert v1.2.0
Expand All @@ -13,7 +13,7 @@ require (
fortio.org/version v1.0.3
github.com/golang/protobuf v1.5.3
github.com/google/uuid v1.4.0
golang.org/x/net v0.18.0
golang.org/x/net v0.19.0
google.golang.org/grpc v1.59.0
)

Expand All @@ -30,10 +30,11 @@ require (
require (
fortio.org/struct2env v0.4.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.15.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
golang.org/x/tools v0.16.0 // indirect
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)
12 changes: 12 additions & 0 deletions go.sum
View file
Open in desktop
Expand Up @@ -27,17 +27,29 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg=
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM=
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
Expand Down