Skip to content

Commit

Permalink
Additional masking functions to be applied to log fields (#87)
Browse files Browse the repository at this point in the history 
* Extending log filtering masking functionality to substrings inside field values

* Wiring up the new functionalities to both the `tflog` and `tfsdklog` packages

* Additional website doc

* Changelog entry

* Fix godoc
  • Loading branch information
Ivan De Marino committed Jul 24, 2022
1 parent 8b0af71 commit b3ff5d7
Show file tree
Hide file tree
Showing 17 changed files with 2,362 additions and 63 deletions.
11 changes: 11 additions & 0 deletions .changelog/87.txt
View file
@@ -0,0 +1,11 @@
```release-note:feature
tflog: Added `MaskAllFieldValuesRegexes()`, `MaskAllFieldValuesStrings()`, `MaskLogRegexes()` and `MaskLogStrings()` functions, which extend further the log masking filtering, for the provider root logger
```

```release-note:feature
tflog: Added `SubsystemMaskAllFieldValuesRegexes()`, `SubsystemMaskAllFieldValuesStrings()`, `SubsystemMaskLogRegexes()` and `SubsystemMaskLogStrings()` functions, which extend further the log masking filtering, for provider subsystem loggers
```

```release-note:feature
tfsdklog: Same functions added to the `tflog` package
```
37 changes: 33 additions & 4 deletions internal/logging/filtering.go
View file
Expand Up @@ -48,6 +48,7 @@ func (lo LoggerOpts) ShouldOmit(msg *string, fieldMaps ...map[string]interface{}
//
// Note that the given input is changed-in-place by this method.
func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{}) {
// Replace any log field value with the corresponding field key equal to the configured strings
if len(lo.MaskFieldValuesWithFieldKeys) > 0 {
for _, k := range lo.MaskFieldValuesWithFieldKeys {
for _, f := range fieldMaps {
Expand All @@ -60,16 +61,44 @@ func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{})
}
}

// Replace any part of the log message matching any of the configured regexp,
// with a masking replacement string
// Replace any part of any log field matching any of the configured regexp
if len(lo.MaskAllFieldValuesRegexes) > 0 {
for _, r := range lo.MaskAllFieldValuesRegexes {
for _, f := range fieldMaps {
for fk, fv := range f {
// Can apply the regexp replacement, only if the field value is indeed a string
fvStr, ok := fv.(string)
if ok {
f[fk] = r.ReplaceAllString(fvStr, logMaskingReplacementString)
}
}
}
}
}

// Replace any part of any log field matching any of the configured strings
if len(lo.MaskAllFieldValuesStrings) > 0 {
for _, s := range lo.MaskAllFieldValuesStrings {
for _, f := range fieldMaps {
for fk, fv := range f {
// Can apply the regexp replacement, only if the field value is indeed a string
fvStr, ok := fv.(string)
if ok {
f[fk] = strings.ReplaceAll(fvStr, s, logMaskingReplacementString)
}
}
}
}
}

// Replace any part of the log message matching any of the configured regexp
if len(lo.MaskMessageRegexes) > 0 {
for _, r := range lo.MaskMessageRegexes {
*msg = r.ReplaceAllString(*msg, logMaskingReplacementString)
}
}

// Replace any part of the log message equal to any of the configured strings,
// with a masking replacement string
// Replace any part of the log message equal to any of the configured strings
if len(lo.MaskMessageStrings) > 0 {
for _, s := range lo.MaskMessageStrings {
*msg = strings.ReplaceAll(*msg, s, logMaskingReplacementString)
Expand Down
44 changes: 44 additions & 0 deletions internal/logging/filtering_test.go
View file
Expand Up @@ -215,6 +215,50 @@ func TestApplyMask(t *testing.T) {
},
},
},
"mask-log-and-fields-matching-regexp": {
lOpts: logging.LoggerOpts{
MaskMessageRegexes: []*regexp.Regexp{
regexp.MustCompile("incorrectly configured BAZ"),
},
MaskAllFieldValuesRegexes: []*regexp.Regexp{
regexp.MustCompile("v1|v2"),
},
},
msg: testLogMsg,
fieldMaps: []map[string]interface{}{
{
"k1": "v1 with some extra text",
"k2": "v2 with more extra text",
},
},
expectedMsg: "System FOO has caused error BAR because of ***",
expectedFieldMaps: []map[string]interface{}{
{
"k1": "*** with some extra text",
"k2": "*** with more extra text",
},
},
},
"mask-log-and-fields-matching-strings": {
lOpts: logging.LoggerOpts{
MaskMessageStrings: []string{"FOO", "BAR", "BAZ"},
MaskAllFieldValuesStrings: []string{"v1", "v2"},
},
msg: testLogMsg,
fieldMaps: []map[string]interface{}{
{
"k1": "v1 with some extra text",
"k2": "v2 with more extra text",
},
},
expectedMsg: "System *** has caused error *** because of incorrectly configured ***",
expectedFieldMaps: []map[string]interface{}{
{
"k1": "*** with some extra text",
"k2": "*** with more extra text",
},
},
},
"mask-log-by-key-and-matching-regexp": {
lOpts: logging.LoggerOpts{
MaskMessageRegexes: []*regexp.Regexp{regexp.MustCompile("incorrectly configured BAZ")},
Expand Down
59 changes: 52 additions & 7 deletions internal/logging/options.go
View file
Expand Up @@ -56,9 +56,9 @@ type LoggerOpts struct {
//
// OmitLogWithFieldKeys = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted
// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted
//
OmitLogWithFieldKeys []string

Expand Down Expand Up @@ -95,12 +95,41 @@ type LoggerOpts struct {
//
// MaskFieldValuesWithFieldKeys = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value
// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value
//
MaskFieldValuesWithFieldKeys []string

// MaskAllFieldValuesRegexes indicates that the logger should replace, within
// all the log field values, the portion matching one of the given *regexp.Regexp.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Example:
//
// MaskAllFieldValuesRegexes = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
MaskAllFieldValuesRegexes []*regexp.Regexp

// MaskAllFieldValuesStrings indicates that the logger should replace, within
// all the log field values, the portion equal to one of the given strings.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Example:
//
// MaskAllFieldValuesStrings = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': 'bar', 'k3': '***' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': '***' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** bar ***' }` -> masked value
MaskAllFieldValuesStrings []string

// MaskMessageRegexes indicates that the logger should replace, within
// a log message, the portion matching one of the given *regexp.Regexp.
//
Expand All @@ -115,7 +144,7 @@ type LoggerOpts struct {
MaskMessageRegexes []*regexp.Regexp

// MaskMessageStrings indicates that the logger should replace, within
// a log message, the portion matching one of the given strings.
// a log message, the portion equal to one of the given strings.
//
// Example:
//
Expand Down Expand Up @@ -266,6 +295,22 @@ func WithMaskFieldValuesWithFieldKeys(keys ...string) Option {
}
}

// WithMaskAllFieldValuesRegexes appends keys to the LoggerOpts.MaskAllFieldValuesRegexes field.
func WithMaskAllFieldValuesRegexes(expressions ...*regexp.Regexp) Option {
return func(l LoggerOpts) LoggerOpts {
l.MaskAllFieldValuesRegexes = append(l.MaskAllFieldValuesRegexes, expressions...)
return l
}
}

// WithMaskAllFieldValuesStrings appends keys to the LoggerOpts.MaskAllFieldValuesStrings field.
func WithMaskAllFieldValuesStrings(matchingStrings ...string) Option {
return func(l LoggerOpts) LoggerOpts {
l.MaskAllFieldValuesStrings = append(l.MaskAllFieldValuesStrings, matchingStrings...)
return l
}
}

// WithMaskMessageRegexes appends *regexp.Regexp to the LoggerOpts.MaskMessageRegexes field.
func WithMaskMessageRegexes(expressions ...*regexp.Regexp) Option {
return func(l LoggerOpts) LoggerOpts {
Expand Down
88 changes: 75 additions & 13 deletions tflog/provider.go
View file
Expand Up @@ -146,9 +146,9 @@ func Error(ctx context.Context, msg string, additionalFields ...map[string]inter
//
// configuration = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted
// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted
//
func OmitLogWithFieldKeys(ctx context.Context, keys ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand Down Expand Up @@ -214,9 +214,9 @@ func OmitLogWithMessageStrings(ctx context.Context, matchingStrings ...string) c
//
// configuration = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value
// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value
//
func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand All @@ -226,9 +226,59 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C
return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskAllFieldValuesRegexes returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all field value substrings,
// matching one of the given *regexp.Regexp.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
//
// Example:
//
// configuration = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
func MaskAllFieldValuesRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)

lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts)

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskAllFieldValuesStrings returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all field value substrings,
// equal to one of the given strings.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
//
// Example:
//
// configuration = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
func MaskAllFieldValuesStrings(ctx context.Context, matchingStrings ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)

lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts)

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskMessageRegexes returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all message substrings matching one
// of the given strings.
// that masks (replaces) with asterisks (`***`) all message substrings,
// matching one of the given *regexp.Regexp.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
Expand All @@ -250,8 +300,8 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
}

// MaskMessageStrings returns a new context.Context that has a modified logger
// that masks (replace) with asterisks (`***`) all message substrings equal to one
// of the given strings.
// that masks (replace) with asterisks (`***`) all message substrings,
// equal to one of the given strings.
//
// Each call to this function is additive:
// the string to mask by are added to the existing configuration.
Expand All @@ -260,9 +310,9 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
//
// configuration = `['foo', 'bar']`
//
// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion
// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is
// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion
// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion
// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is
// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion
//
func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand All @@ -271,3 +321,15 @@ func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskLogRegexes is a shortcut to invoke MaskMessageRegexes and MaskAllFieldValuesRegexes using the same input.
// Refer to those functions for details.
func MaskLogRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
return MaskMessageRegexes(MaskAllFieldValuesRegexes(ctx, expressions...), expressions...)
}

// MaskLogStrings is a shortcut to invoke MaskMessageStrings and MaskAllFieldValuesStrings using the same input.
// Refer to those functions for details.
func MaskLogStrings(ctx context.Context, matchingStrings ...string) context.Context {
return MaskMessageStrings(MaskAllFieldValuesStrings(ctx, matchingStrings...), matchingStrings...)
}

0 comments on commit b3ff5d7

Please sign in to comment.