From 36002e290aa503e4575928fc0edb0ff947c3b790 Mon Sep 17 00:00:00 2001 From: Ale Ornelas Figueroa Date: Mon, 30 May 2022 20:04:08 -0500 Subject: [PATCH] LogpushFilter support (#915) Add support for LogpushJob filter field --- logpush.go | 141 ++++++++++++++++++++++++++++++++++++---- logpush_example_test.go | 26 ++++++++ logpush_test.go | 93 ++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 11 deletions(-) diff --git a/logpush.go b/logpush.go index 1f20ddd2d3..c7aaeae08a 100644 --- a/logpush.go +++ b/logpush.go @@ -12,17 +12,51 @@ import ( // LogpushJob describes a Logpush job. type LogpushJob struct { - ID int `json:"id,omitempty"` - Dataset string `json:"dataset"` - Enabled bool `json:"enabled"` - Name string `json:"name"` - LogpullOptions string `json:"logpull_options"` - DestinationConf string `json:"destination_conf"` - OwnershipChallenge string `json:"ownership_challenge,omitempty"` - LastComplete *time.Time `json:"last_complete,omitempty"` - LastError *time.Time `json:"last_error,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - Frequency string `json:"frequency,omitempty"` + ID int `json:"id,omitempty"` + Dataset string `json:"dataset"` + Enabled bool `json:"enabled"` + Name string `json:"name"` + LogpullOptions string `json:"logpull_options"` + DestinationConf string `json:"destination_conf"` + OwnershipChallenge string `json:"ownership_challenge,omitempty"` + LastComplete *time.Time `json:"last_complete,omitempty"` + LastError *time.Time `json:"last_error,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + Frequency string `json:"frequency,omitempty"` + Filter LogpushJobFilters `json:"filter,omitempty"` +} + +type LogpushJobFilters struct { + Where LogpushJobFilter `json:"where"` +} + +type Operator string + +const ( + Equal Operator = "eq" + NotEqual Operator = "!eq" + LessThan Operator = "lt" + LessThanOrEqual Operator = "lte" + GreaterThan Operator = "gt" + GreaterThanOrEqual Operator = "gte" + StartsWith Operator = "startsWith" + EndsWith Operator = "endsWith" + NotStartsWith Operator = "!startsWith" + NotEndsWith Operator = "!endsWith" + Contains Operator = "contains" + NotContains Operator = "!contains" + ValueIsIn Operator = "in" + ValueIsNotIn Operator = "!in" +) + +type LogpushJobFilter struct { + // either this + And []LogpushJobFilter `json:"and,omitempty"` + Or []LogpushJobFilter `json:"or,omitempty"` + // or this + Key string `json:"key,omitempty"` + Operator Operator `json:"operator,omitempty"` + Value interface{} `json:"value,omitempty"` } // LogpushJobsResponse is the API response, containing an array of Logpush Jobs. @@ -93,6 +127,91 @@ type LogpushDestinationExistsRequest struct { DestinationConf string `json:"destination_conf"` } +// Custom Marshaller for LogpushJob filter key. +func (f LogpushJob) MarshalJSON() ([]byte, error) { + type Alias LogpushJob + + filter, err := json.Marshal(f.Filter) + + if err != nil { + return nil, err + } + + return json.Marshal(&struct { + Filter string `json:"filter,omitempty"` + Alias + }{ + Filter: string(filter), + Alias: (Alias)(f), + }) +} + +// Custom Unmarshaller for LogpushJob filter key. +func (f *LogpushJob) UnmarshalJSON(data []byte) error { + type Alias LogpushJob + aux := &struct { + Filter string `json:"filter,omitempty"` + *Alias + }{ + Alias: (*Alias)(f), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + if aux != nil && aux.Filter != "" { + var filter LogpushJobFilters + if err := json.Unmarshal([]byte(aux.Filter), &filter); err != nil { + return err + } + if err := filter.Where.Validate(); err != nil { + return err + } + f.Filter = filter + } + return nil +} + +func (filter *LogpushJobFilter) Validate() error { + if filter.And != nil { + if filter.Or != nil || filter.Key != "" || filter.Operator != "" || filter.Value != nil { + return errors.New("And can't be set with Or, Key, Operator or Value") + } + for i, element := range filter.And { + err := element.Validate() + if err != nil { + return errors.WithMessagef(err, "element %v in And is invalid", i) + } + } + return nil + } + if filter.Or != nil { + if filter.And != nil || filter.Key != "" || filter.Operator != "" || filter.Value != nil { + return errors.New("Or can't be set with And, Key, Operator or Value") + } + for i, element := range filter.Or { + err := element.Validate() + if err != nil { + return errors.WithMessagef(err, "element %v in Or is invalid", i) + } + } + return nil + } + if filter.Key == "" { + return errors.New("Key is missing") + } + + if filter.Operator == "" { + return errors.New("Operator is missing") + } + + if filter.Value == nil { + return errors.New("Value is missing") + } + + return nil +} + // CreateAccountLogpushJob creates a new account-level Logpush Job. // // API reference: https://api.cloudflare.com/#logpush-jobs-create-logpush-job diff --git a/logpush_example_test.go b/logpush_example_test.go index cf4ef3e629..db56b82c0c 100644 --- a/logpush_example_test.go +++ b/logpush_example_test.go @@ -2,6 +2,7 @@ package cloudflare_test import ( "context" + "encoding/json" "fmt" "log" @@ -172,3 +173,28 @@ func ExampleAPI_CheckZoneLogpushDestinationExists() { fmt.Printf("%+v\n", exists) } + +func ExampleLogpushJob_MarshalJSON() { + job := cloudflare.LogpushJob{ + Name: "example.com static assets", + LogpullOptions: "fields=RayID,ClientIP,EdgeStartTimestamp×tamps=rfc3339&CVE-2021-44228=true", + Dataset: "http_requests", + DestinationConf: "s3://?region=us-west-2/", + Filter: cloudflare.LogpushJobFilters{ + Where: cloudflare.LogpushJobFilter{ + And: []cloudflare.LogpushJobFilter{ + {Key: "ClientRequestPath", Operator: cloudflare.Contains, Value: "/static\\"}, + {Key: "ClientRequestHost", Operator: cloudflare.Equal, Value: "example.com"}, + }, + }, + }, + } + + jobstring, err := json.Marshal(job) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", jobstring) + // Output: {"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\",\"value\":\"/static\\\\\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"} +} diff --git a/logpush_test.go b/logpush_test.go index f2df74141e..f8265dfac1 100644 --- a/logpush_test.go +++ b/logpush_test.go @@ -2,7 +2,9 @@ package cloudflare import ( "context" + "encoding/json" "fmt" + "log" "net/http" "strconv" "testing" @@ -330,3 +332,94 @@ func TestCheckLogpushDestinationExists(t *testing.T) { }) } } + +var ( + validFilter LogpushJobFilter = LogpushJobFilter{Key: "ClientRequestPath", Operator: Contains, Value: "static"} +) + +var logpushJobFiltersTest = []struct { + name string + input LogpushJobFilter + haserror bool + expectedErrorMessage string +}{ + // Tests without And or Or + {"Empty Filter", LogpushJobFilter{}, true, "Key is missing"}, + {"Missing Operator", LogpushJobFilter{Key: "ClientRequestPath"}, true, "Operator is missing"}, + {"Missing Value", LogpushJobFilter{Key: "ClientRequestPath", Operator: Contains}, true, "Value is missing"}, + {"Valid Basic Filter", validFilter, false, ""}, + // Tests with And + {"Valid And Filter", LogpushJobFilter{And: []LogpushJobFilter{validFilter}}, false, ""}, + {"And and Or", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Or: []LogpushJobFilter{validFilter}}, true, "And can't be set with Or, Key, Operator or Value"}, + {"And and Key", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Key: "Key"}, true, "And can't be set with Or, Key, Operator or Value"}, + {"And and Operator", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Operator: Contains}, true, "And can't be set with Or, Key, Operator or Value"}, + {"And and Value", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Value: "Value"}, true, "And can't be set with Or, Key, Operator or Value"}, + {"And with nested error", LogpushJobFilter{And: []LogpushJobFilter{validFilter, {}}}, true, "element 1 in And is invalid: Key is missing"}, + // Tests with Or + {"Valid Or Filter", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}}, false, ""}, + {"Or and Key", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Key: "Key"}, true, "Or can't be set with And, Key, Operator or Value"}, + {"Or and Operator", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Operator: Contains}, true, "Or can't be set with And, Key, Operator or Value"}, + {"Or and Value", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Value: "Value"}, true, "Or can't be set with And, Key, Operator or Value"}, + {"Or with nested error", LogpushJobFilter{Or: []LogpushJobFilter{validFilter, {}}}, true, "element 1 in Or is invalid: Key is missing"}, +} + +func TestLogpushJobFilter_Validate(t *testing.T) { + for _, tt := range logpushJobFiltersTest { + t.Run(tt.name, func(t *testing.T) { + got := tt.input.Validate() + if tt.haserror { + assert.ErrorContains(t, got, tt.expectedErrorMessage) + } else { + assert.NoError(t, got) + } + }) + } +} + +func TestLogpushJob_Unmarshall(t *testing.T) { + t.Run("Valid Filter", func(t *testing.T) { + jsonstring := `{"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\",\"value\":\"/static\\\\\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}` + var job LogpushJob + if err := json.Unmarshal([]byte(jsonstring), &job); err != nil { + log.Fatal(err) + } + + assert.Equal(t, LogpushJob{ + Name: "example.com static assets", + LogpullOptions: "fields=RayID,ClientIP,EdgeStartTimestamp×tamps=rfc3339&CVE-2021-44228=true", + Dataset: "http_requests", + DestinationConf: "s3://?region=us-west-2/", + Filter: LogpushJobFilters{ + Where: LogpushJobFilter{ + And: []LogpushJobFilter{ + {Key: "ClientRequestPath", Operator: Contains, Value: "/static\\"}, + {Key: "ClientRequestHost", Operator: Equal, Value: "example.com"}, + }, + }, + }, + }, job) + }) + + t.Run("Invalid Filter", func(t *testing.T) { + jsonstring := `{"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}` + var job LogpushJob + err := json.Unmarshal([]byte(jsonstring), &job) + + assert.ErrorContains(t, err, "element 0 in And is invalid: Value is missing") + }) + + t.Run("No Filter", func(t *testing.T) { + jsonstring := `{"dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}` + var job LogpushJob + if err := json.Unmarshal([]byte(jsonstring), &job); err != nil { + log.Fatal(err) + } + + assert.Equal(t, LogpushJob{ + Name: "example.com static assets", + LogpullOptions: "fields=RayID,ClientIP,EdgeStartTimestamp×tamps=rfc3339&CVE-2021-44228=true", + Dataset: "http_requests", + DestinationConf: "s3://?region=us-west-2/", + }, job) + }) +}