add http.send request attribute to ignore headers for caching key

Signed-off-by: Rudrakh Panigrahi <rudrakh97@gmail.com>
open-policy-agent · Apr 4, 2024 · 1c9b35d · 1c9b35d
1 parent 8fde826
commit 1c9b35d
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 2 deletions.
diff --git a/topdown/http.go b/topdown/http.go
@@ -68,6 +68,7 @@ var allowedKeyNames = [...]string{
 	"raise_error",
 	"caching_mode",
 	"max_retry_attempts",
+	"cache_ignored_headers",
 }
 
 // ref: https://www.rfc-editor.org/rfc/rfc7231#section-6.1
@@ -168,7 +169,11 @@ func getHTTPResponse(bctx BuiltinContext, req ast.Object) (*ast.Term, error) {
 
 	bctx.Metrics.Timer(httpSendLatencyMetricKey).Start()
 
-	reqExecutor, err := newHTTPRequestExecutor(bctx, req)
+	key, err := getKeyFromRequest(req)
+	if err != nil {
+		return nil, err
+	}
+	reqExecutor, err := newHTTPRequestExecutor(bctx, key)
 	if err != nil {
 		return nil, err
 	}
@@ -198,6 +203,36 @@ func getHTTPResponse(bctx BuiltinContext, req ast.Object) (*ast.Term, error) {
 	return ast.NewTerm(resp), nil
 }
 
+// getKeyFromRequest returns a key to be used for caching HTTP responses
+// deletes headers from request object mentioned in cache_ignored_headers
+func getKeyFromRequest(req ast.Object) (ast.Object, error) {
+	var cacheIgnoredHeaders []string
+	var allHeaders map[string]interface{}
+	cacheIgnoredHeadersTerm := req.Get(ast.StringTerm("cache_ignored_headers"))
+	allHeadersTerm := req.Get(ast.StringTerm("headers"))
+	if cacheIgnoredHeadersTerm != nil && allHeadersTerm != nil {
+		err := ast.As(cacheIgnoredHeadersTerm.Value, &cacheIgnoredHeaders)
+		if err != nil {
+			return nil, err
+		}
+		err = ast.As(allHeadersTerm.Value, &allHeaders)
+		if err != nil {
+			return nil, err
+		}
+		for _, header := range cacheIgnoredHeaders {
+			delete(allHeaders, header)
+		}
+		val, err := ast.InterfaceToValue(allHeaders)
+		if err != nil {
+			return nil, err
+		}
+		allHeadersTerm.Value = val
+		req.Insert(ast.StringTerm("headers"), allHeadersTerm)
+	}
+	req.Insert(ast.StringTerm("cache_ignored_headers"), ast.NullTerm())
+	return req, nil
+}
+
 func init() {
 	createAllowedKeys()
 	createCacheableHTTPStatusCodes()
@@ -482,7 +517,7 @@ func createHTTPRequest(bctx BuiltinContext, obj ast.Object) (*http.Request, *htt
 		case "cache", "caching_mode",
 			"force_cache", "force_cache_duration_seconds",
 			"force_json_decode", "force_yaml_decode",
-			"raise_error", "max_retry_attempts": // no-op
+			"raise_error", "max_retry_attempts", "cache_ignored_headers": // no-op
 		default:
 			return nil, nil, fmt.Errorf("invalid parameter %q", key)
 		}

diff --git a/topdown/http_test.go b/topdown/http_test.go
@@ -1012,6 +1012,46 @@ func TestHTTPSendCaching(t *testing.T) {
 			response:         `{"x": 1}`,
 			expectedReqCount: 3,
 		},
+		{
+			note: "http.send GET different headers but still cached because ignored",
+			ruleTemplate: `p = x {
+									r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "cache_ignored_headers": ["h2"]})
+									r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "cache_ignored_headers": ["h2"]}) # cached
+									x = r1.body
+								}`,
+			response:         `{"x": 1}`,
+			expectedReqCount: 1,
+		},
+		{
+			note: "http.send GET cache miss different headers (force_cache enabled)",
+			ruleTemplate: `p = x {
+									r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300})
+									r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "force_cache": true, "force_cache_duration_seconds": 300})
+									x = r1.body
+								}`,
+			response:         `{"x": 1}`,
+			expectedReqCount: 2,
+		},
+		{
+			note: "http.send GET different headers but still cached because ignored (force_cache enabled)",
+			ruleTemplate: `p = x {
+									r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]})
+									r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]}) # cached
+									x = r1.body
+								}`,
+			response:         `{"x": 1}`,
+			expectedReqCount: 1,
+		},
+		{
+			note: "http.send GET different cache_ignored_headers but still cached (force_cache enabled)",
+			ruleTemplate: `p = x {
+									r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]})
+									r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2", "h3"]}) # cached
+									x = r1.body
+								}`,
+			response:         `{"x": 1}`,
+			expectedReqCount: 1,
+		},
 		{
 			note: "http.send POST cache miss different body",
 			ruleTemplate: `p = x {