LogpushFilter support (#915)

Add support for LogpushJob filter field
cloudflare · May 31, 2022 · 36002e2 · 36002e2
1 parent 5e256ea
commit 36002e2
Show file tree

Hide file tree

Showing 3 changed files with 249 additions and 11 deletions.
diff --git a/logpush.go b/logpush.go
@@ -12,17 +12,51 @@ import (
 
 // LogpushJob describes a Logpush job.
 type LogpushJob struct {
-	ID                 int        `json:"id,omitempty"`
-	Dataset            string     `json:"dataset"`
-	Enabled            bool       `json:"enabled"`
-	Name               string     `json:"name"`
-	LogpullOptions     string     `json:"logpull_options"`
-	DestinationConf    string     `json:"destination_conf"`
-	OwnershipChallenge string     `json:"ownership_challenge,omitempty"`
-	LastComplete       *time.Time `json:"last_complete,omitempty"`
-	LastError          *time.Time `json:"last_error,omitempty"`
-	ErrorMessage       string     `json:"error_message,omitempty"`
-	Frequency          string     `json:"frequency,omitempty"`
+	ID                 int               `json:"id,omitempty"`
+	Dataset            string            `json:"dataset"`
+	Enabled            bool              `json:"enabled"`
+	Name               string            `json:"name"`
+	LogpullOptions     string            `json:"logpull_options"`
+	DestinationConf    string            `json:"destination_conf"`
+	OwnershipChallenge string            `json:"ownership_challenge,omitempty"`
+	LastComplete       *time.Time        `json:"last_complete,omitempty"`
+	LastError          *time.Time        `json:"last_error,omitempty"`
+	ErrorMessage       string            `json:"error_message,omitempty"`
+	Frequency          string            `json:"frequency,omitempty"`
+	Filter             LogpushJobFilters `json:"filter,omitempty"`
+}
+
+type LogpushJobFilters struct {
+	Where LogpushJobFilter `json:"where"`
+}
+
+type Operator string
+
+const (
+	Equal              Operator = "eq"
+	NotEqual           Operator = "!eq"
+	LessThan           Operator = "lt"
+	LessThanOrEqual    Operator = "lte"
+	GreaterThan        Operator = "gt"
+	GreaterThanOrEqual Operator = "gte"
+	StartsWith         Operator = "startsWith"
+	EndsWith           Operator = "endsWith"
+	NotStartsWith      Operator = "!startsWith"
+	NotEndsWith        Operator = "!endsWith"
+	Contains           Operator = "contains"
+	NotContains        Operator = "!contains"
+	ValueIsIn          Operator = "in"
+	ValueIsNotIn       Operator = "!in"
+)
+
+type LogpushJobFilter struct {
+	// either this
+	And []LogpushJobFilter `json:"and,omitempty"`
+	Or  []LogpushJobFilter `json:"or,omitempty"`
+	// or this
+	Key      string      `json:"key,omitempty"`
+	Operator Operator    `json:"operator,omitempty"`
+	Value    interface{} `json:"value,omitempty"`
 }
 
 // LogpushJobsResponse is the API response, containing an array of Logpush Jobs.
@@ -93,6 +127,91 @@ type LogpushDestinationExistsRequest struct {
 	DestinationConf string `json:"destination_conf"`
 }
 
+// Custom Marshaller for LogpushJob filter key.
+func (f LogpushJob) MarshalJSON() ([]byte, error) {
+	type Alias LogpushJob
+
+	filter, err := json.Marshal(f.Filter)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(&struct {
+		Filter string `json:"filter,omitempty"`
+		Alias
+	}{
+		Filter: string(filter),
+		Alias:  (Alias)(f),
+	})
+}
+
+// Custom Unmarshaller for LogpushJob filter key.
+func (f *LogpushJob) UnmarshalJSON(data []byte) error {
+	type Alias LogpushJob
+	aux := &struct {
+		Filter string `json:"filter,omitempty"`
+		*Alias
+	}{
+		Alias: (*Alias)(f),
+	}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	if aux != nil && aux.Filter != "" {
+		var filter LogpushJobFilters
+		if err := json.Unmarshal([]byte(aux.Filter), &filter); err != nil {
+			return err
+		}
+		if err := filter.Where.Validate(); err != nil {
+			return err
+		}
+		f.Filter = filter
+	}
+	return nil
+}
+
+func (filter *LogpushJobFilter) Validate() error {
+	if filter.And != nil {
+		if filter.Or != nil || filter.Key != "" || filter.Operator != "" || filter.Value != nil {
+			return errors.New("And can't be set with Or, Key, Operator or Value")
+		}
+		for i, element := range filter.And {
+			err := element.Validate()
+			if err != nil {
+				return errors.WithMessagef(err, "element %v in And is invalid", i)
+			}
+		}
+		return nil
+	}
+	if filter.Or != nil {
+		if filter.And != nil || filter.Key != "" || filter.Operator != "" || filter.Value != nil {
+			return errors.New("Or can't be set with And, Key, Operator or Value")
+		}
+		for i, element := range filter.Or {
+			err := element.Validate()
+			if err != nil {
+				return errors.WithMessagef(err, "element %v in Or is invalid", i)
+			}
+		}
+		return nil
+	}
+	if filter.Key == "" {
+		return errors.New("Key is missing")
+	}
+
+	if filter.Operator == "" {
+		return errors.New("Operator is missing")
+	}
+
+	if filter.Value == nil {
+		return errors.New("Value is missing")
+	}
+
+	return nil
+}
+
 // CreateAccountLogpushJob creates a new account-level Logpush Job.
 //
 // API reference: https://api.cloudflare.com/#logpush-jobs-create-logpush-job

diff --git a/logpush_example_test.go b/logpush_example_test.go
@@ -2,6 +2,7 @@ package cloudflare_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 
@@ -172,3 +173,28 @@ func ExampleAPI_CheckZoneLogpushDestinationExists() {
 
 	fmt.Printf("%+v\n", exists)
 }
+
+func ExampleLogpushJob_MarshalJSON() {
+	job := cloudflare.LogpushJob{
+		Name:            "example.com static assets",
+		LogpullOptions:  "fields=RayID,ClientIP,EdgeStartTimestamp&timestamps=rfc3339&CVE-2021-44228=true",
+		Dataset:         "http_requests",
+		DestinationConf: "s3://<BUCKET_PATH>?region=us-west-2/",
+		Filter: cloudflare.LogpushJobFilters{
+			Where: cloudflare.LogpushJobFilter{
+				And: []cloudflare.LogpushJobFilter{
+					{Key: "ClientRequestPath", Operator: cloudflare.Contains, Value: "/static\\"},
+					{Key: "ClientRequestHost", Operator: cloudflare.Equal, Value: "example.com"},
+				},
+			},
+		},
+	}
+
+	jobstring, err := json.Marshal(job)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("%s", jobstring)
+	// Output: {"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\",\"value\":\"/static\\\\\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}
+}
diff --git a/logpush_test.go b/logpush_test.go
@@ -2,7 +2,9 @@ package cloudflare
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"strconv"
 	"testing"
@@ -330,3 +332,94 @@ func TestCheckLogpushDestinationExists(t *testing.T) {
 		})
 	}
 }
+
+var (
+	validFilter LogpushJobFilter = LogpushJobFilter{Key: "ClientRequestPath", Operator: Contains, Value: "static"}
+)
+
+var logpushJobFiltersTest = []struct {
+	name                 string
+	input                LogpushJobFilter
+	haserror             bool
+	expectedErrorMessage string
+}{
+	// Tests without And or Or
+	{"Empty Filter", LogpushJobFilter{}, true, "Key is missing"},
+	{"Missing Operator", LogpushJobFilter{Key: "ClientRequestPath"}, true, "Operator is missing"},
+	{"Missing Value", LogpushJobFilter{Key: "ClientRequestPath", Operator: Contains}, true, "Value is missing"},
+	{"Valid Basic Filter", validFilter, false, ""},
+	// Tests with And
+	{"Valid And Filter", LogpushJobFilter{And: []LogpushJobFilter{validFilter}}, false, ""},
+	{"And and Or", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Or: []LogpushJobFilter{validFilter}}, true, "And can't be set with Or, Key, Operator or Value"},
+	{"And and Key", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Key: "Key"}, true, "And can't be set with Or, Key, Operator or Value"},
+	{"And and Operator", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Operator: Contains}, true, "And can't be set with Or, Key, Operator or Value"},
+	{"And and Value", LogpushJobFilter{And: []LogpushJobFilter{validFilter}, Value: "Value"}, true, "And can't be set with Or, Key, Operator or Value"},
+	{"And with nested error", LogpushJobFilter{And: []LogpushJobFilter{validFilter, {}}}, true, "element 1 in And is invalid: Key is missing"},
+	// Tests with Or
+	{"Valid Or Filter", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}}, false, ""},
+	{"Or and Key", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Key: "Key"}, true, "Or can't be set with And, Key, Operator or Value"},
+	{"Or and Operator", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Operator: Contains}, true, "Or can't be set with And, Key, Operator or Value"},
+	{"Or and Value", LogpushJobFilter{Or: []LogpushJobFilter{validFilter}, Value: "Value"}, true, "Or can't be set with And, Key, Operator or Value"},
+	{"Or with nested error", LogpushJobFilter{Or: []LogpushJobFilter{validFilter, {}}}, true, "element 1 in Or is invalid: Key is missing"},
+}
+
+func TestLogpushJobFilter_Validate(t *testing.T) {
+	for _, tt := range logpushJobFiltersTest {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.input.Validate()
+			if tt.haserror {
+				assert.ErrorContains(t, got, tt.expectedErrorMessage)
+			} else {
+				assert.NoError(t, got)
+			}
+		})
+	}
+}
+
+func TestLogpushJob_Unmarshall(t *testing.T) {
+	t.Run("Valid Filter", func(t *testing.T) {
+		jsonstring := `{"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\",\"value\":\"/static\\\\\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}`
+		var job LogpushJob
+		if err := json.Unmarshal([]byte(jsonstring), &job); err != nil {
+			log.Fatal(err)
+		}
+
+		assert.Equal(t, LogpushJob{
+			Name:            "example.com static assets",
+			LogpullOptions:  "fields=RayID,ClientIP,EdgeStartTimestamp&timestamps=rfc3339&CVE-2021-44228=true",
+			Dataset:         "http_requests",
+			DestinationConf: "s3://<BUCKET_PATH>?region=us-west-2/",
+			Filter: LogpushJobFilters{
+				Where: LogpushJobFilter{
+					And: []LogpushJobFilter{
+						{Key: "ClientRequestPath", Operator: Contains, Value: "/static\\"},
+						{Key: "ClientRequestHost", Operator: Equal, Value: "example.com"},
+					},
+				},
+			},
+		}, job)
+	})
+
+	t.Run("Invalid Filter", func(t *testing.T) {
+		jsonstring := `{"filter":"{\"where\":{\"and\":[{\"key\":\"ClientRequestPath\",\"operator\":\"contains\"},{\"key\":\"ClientRequestHost\",\"operator\":\"eq\",\"value\":\"example.com\"}]}}","dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}`
+		var job LogpushJob
+		err := json.Unmarshal([]byte(jsonstring), &job)
+
+		assert.ErrorContains(t, err, "element 0 in And is invalid: Value is missing")
+	})
+
+	t.Run("No Filter", func(t *testing.T) {
+		jsonstring := `{"dataset":"http_requests","enabled":false,"name":"example.com static assets","logpull_options":"fields=RayID,ClientIP,EdgeStartTimestamp\u0026timestamps=rfc3339\u0026CVE-2021-44228=true","destination_conf":"s3://\u003cBUCKET_PATH\u003e?region=us-west-2/"}`
+		var job LogpushJob
+		if err := json.Unmarshal([]byte(jsonstring), &job); err != nil {
+			log.Fatal(err)
+		}
+
+		assert.Equal(t, LogpushJob{
+			Name:            "example.com static assets",
+			LogpullOptions:  "fields=RayID,ClientIP,EdgeStartTimestamp&timestamps=rfc3339&CVE-2021-44228=true",
+			Dataset:         "http_requests",
+			DestinationConf: "s3://<BUCKET_PATH>?region=us-west-2/",
+		}, job)
+	})
+}