Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Additional masking functions to be applied to log fields #87

Merged
merged 6 commits into from Jul 24, 2022
Merged

Additional masking functions to be applied to log fields #87

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8235671
Extending log filtering masking functionality to substrings inside fi…
Jul 22, 2022
2b75f66
Wiring up the new functionalities to both the `tflog` and `tfsdklog` …
Jul 22, 2022
85181be
Additional website doc
Jul 22, 2022
bc51462
Changelog entry
Jul 22, 2022
d06e6a3
Fix godoc
Jul 22, 2022
03b4ad4
Fix elided fields format in godoc
Jul 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 11 additions & 0 deletions .changelog/87.txt
View file
Open in desktop
@@ -0,0 +1,11 @@
```release-note:feature
tflog: Added `MaskAllFieldValuesRegexes()`, `MaskAllFieldValuesStrings()`, `MaskLogRegexes()` and `MaskLogStrings()` functions, which extend further the log masking filtering, for the provider root logger
```

```release-note:feature
tflog: Added `SubsystemMaskAllFieldValuesRegexes()`, `SubsystemMaskAllFieldValuesStrings()`, `SubsystemMaskLogRegexes()` and `SubsystemMaskLogStrings()` functions, which extend further the log masking filtering, for provider subsystem loggers
```

```release-note:feature
tfsdklog: Same functions added to the `tflog` package
```
37 changes: 33 additions & 4 deletions internal/logging/filtering.go
View file
Open in desktop
Expand Up @@ -48,6 +48,7 @@ func (lo LoggerOpts) ShouldOmit(msg *string, fieldMaps ...map[string]interface{}
//
// Note that the given input is changed-in-place by this method.
func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{}) {
// Replace any log field value with the corresponding field key equal to the configured strings
if len(lo.MaskFieldValuesWithFieldKeys) > 0 {
for _, k := range lo.MaskFieldValuesWithFieldKeys {
for _, f := range fieldMaps {
Expand All @@ -60,16 +61,44 @@ func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{})
}
}

// Replace any part of the log message matching any of the configured regexp,
// with a masking replacement string
// Replace any part of any log field matching any of the configured regexp
if len(lo.MaskAllFieldValuesRegexes) > 0 {
for _, r := range lo.MaskAllFieldValuesRegexes {
for _, f := range fieldMaps {
for fk, fv := range f {
// Can apply the regexp replacement, only if the field value is indeed a string
fvStr, ok := fv.(string)
detro marked this conversation as resolved.
Show resolved Hide resolved
if ok {
f[fk] = r.ReplaceAllString(fvStr, logMaskingReplacementString)
}
}
}
}
}

// Replace any part of any log field matching any of the configured strings
if len(lo.MaskAllFieldValuesStrings) > 0 {
for _, s := range lo.MaskAllFieldValuesStrings {
for _, f := range fieldMaps {
for fk, fv := range f {
// Can apply the regexp replacement, only if the field value is indeed a string
fvStr, ok := fv.(string)
if ok {
f[fk] = strings.ReplaceAll(fvStr, s, logMaskingReplacementString)
}
}
}
}
}

// Replace any part of the log message matching any of the configured regexp
if len(lo.MaskMessageRegexes) > 0 {
for _, r := range lo.MaskMessageRegexes {
*msg = r.ReplaceAllString(*msg, logMaskingReplacementString)
}
}

// Replace any part of the log message equal to any of the configured strings,
// with a masking replacement string
// Replace any part of the log message equal to any of the configured strings
if len(lo.MaskMessageStrings) > 0 {
for _, s := range lo.MaskMessageStrings {
*msg = strings.ReplaceAll(*msg, s, logMaskingReplacementString)
Expand Down
44 changes: 44 additions & 0 deletions internal/logging/filtering_test.go
View file
Open in desktop
Expand Up @@ -215,6 +215,50 @@ func TestApplyMask(t *testing.T) {
},
},
},
"mask-log-and-fields-matching-regexp": {
lOpts: logging.LoggerOpts{
MaskMessageRegexes: []*regexp.Regexp{
regexp.MustCompile("incorrectly configured BAZ"),
},
MaskAllFieldValuesRegexes: []*regexp.Regexp{
regexp.MustCompile("v1|v2"),
},
},
msg: testLogMsg,
fieldMaps: []map[string]interface{}{
{
"k1": "v1 with some extra text",
"k2": "v2 with more extra text",
},
},
expectedMsg: "System FOO has caused error BAR because of ***",
expectedFieldMaps: []map[string]interface{}{
{
"k1": "*** with some extra text",
"k2": "*** with more extra text",
},
},
},
"mask-log-and-fields-matching-strings": {
lOpts: logging.LoggerOpts{
MaskMessageStrings: []string{"FOO", "BAR", "BAZ"},
MaskAllFieldValuesStrings: []string{"v1", "v2"},
},
msg: testLogMsg,
fieldMaps: []map[string]interface{}{
{
"k1": "v1 with some extra text",
"k2": "v2 with more extra text",
},
},
expectedMsg: "System *** has caused error *** because of incorrectly configured ***",
expectedFieldMaps: []map[string]interface{}{
{
"k1": "*** with some extra text",
"k2": "*** with more extra text",
},
},
},
"mask-log-by-key-and-matching-regexp": {
lOpts: logging.LoggerOpts{
MaskMessageRegexes: []*regexp.Regexp{regexp.MustCompile("incorrectly configured BAZ")},
Expand Down
59 changes: 52 additions & 7 deletions internal/logging/options.go
View file
Open in desktop
Expand Up @@ -56,9 +56,9 @@ type LoggerOpts struct {
//
// OmitLogWithFieldKeys = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted
// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted
//
OmitLogWithFieldKeys []string

Expand Down Expand Up @@ -95,12 +95,41 @@ type LoggerOpts struct {
//
// MaskFieldValuesWithFieldKeys = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value
// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value
//
MaskFieldValuesWithFieldKeys []string

// MaskAllFieldValuesRegexes indicates that the logger should replace, within
// all the log field values, the portion matching one of the given *regexp.Regexp.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Example:
//
// MaskAllFieldValuesRegexes = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
MaskAllFieldValuesRegexes []*regexp.Regexp

// MaskAllFieldValuesStrings indicates that the logger should replace, within
// all the log field values, the portion equal to one of the given strings.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Example:
//
// MaskAllFieldValuesStrings = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': 'bar', 'k3': '***' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': '***' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** bar ***' }` -> masked value
MaskAllFieldValuesStrings []string

// MaskMessageRegexes indicates that the logger should replace, within
// a log message, the portion matching one of the given *regexp.Regexp.
//
Expand All @@ -115,7 +144,7 @@ type LoggerOpts struct {
MaskMessageRegexes []*regexp.Regexp

// MaskMessageStrings indicates that the logger should replace, within
// a log message, the portion matching one of the given strings.
// a log message, the portion equal to one of the given strings.
//
// Example:
//
Expand Down Expand Up @@ -266,6 +295,22 @@ func WithMaskFieldValuesWithFieldKeys(keys ...string) Option {
}
}

// WithMaskAllFieldValuesRegexes appends keys to the LoggerOpts.MaskAllFieldValuesRegexes field.
func WithMaskAllFieldValuesRegexes(expressions ...*regexp.Regexp) Option {
return func(l LoggerOpts) LoggerOpts {
l.MaskAllFieldValuesRegexes = append(l.MaskAllFieldValuesRegexes, expressions...)
return l
}
}

// WithMaskAllFieldValuesStrings appends keys to the LoggerOpts.MaskAllFieldValuesStrings field.
func WithMaskAllFieldValuesStrings(matchingStrings ...string) Option {
return func(l LoggerOpts) LoggerOpts {
l.MaskAllFieldValuesStrings = append(l.MaskAllFieldValuesStrings, matchingStrings...)
return l
}
}

// WithMaskMessageRegexes appends *regexp.Regexp to the LoggerOpts.MaskMessageRegexes field.
func WithMaskMessageRegexes(expressions ...*regexp.Regexp) Option {
return func(l LoggerOpts) LoggerOpts {
Expand Down
88 changes: 75 additions & 13 deletions tflog/provider.go
View file
Open in desktop
Expand Up @@ -146,9 +146,9 @@ func Error(ctx context.Context, msg string, additionalFields ...map[string]inter
//
// configuration = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted
// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed
// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted
//
func OmitLogWithFieldKeys(ctx context.Context, keys ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand Down Expand Up @@ -214,9 +214,9 @@ func OmitLogWithMessageStrings(ctx context.Context, matchingStrings ...string) c
//
// configuration = `['foo', 'baz']`
//
// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value
// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value
// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value
// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value
//
func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand All @@ -226,9 +226,59 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C
return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskAllFieldValuesRegexes returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all field value substrings,
// matching one of the given *regexp.Regexp.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
//
// Example:
//
// configuration = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
func MaskAllFieldValuesRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)

lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts)

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskAllFieldValuesStrings returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all field value substrings,
// equal to one of the given strings.
//
// Note that the replacement will happen, only for field values that are of type string.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
//
// Example:
//
// configuration = `[regexp.MustCompile("(foo|bar)")]`
//
// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value
// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value
// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value
//
func MaskAllFieldValuesStrings(ctx context.Context, matchingStrings ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)

lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts)

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskMessageRegexes returns a new context.Context that has a modified logger
// that masks (replaces) with asterisks (`***`) all message substrings matching one
// of the given strings.
// that masks (replaces) with asterisks (`***`) all message substrings,
// matching one of the given *regexp.Regexp.
//
// Each call to this function is additive:
// the regexp to mask by are added to the existing configuration.
Expand All @@ -250,8 +300,8 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
}

// MaskMessageStrings returns a new context.Context that has a modified logger
// that masks (replace) with asterisks (`***`) all message substrings equal to one
// of the given strings.
// that masks (replace) with asterisks (`***`) all message substrings,
// equal to one of the given strings.
//
// Each call to this function is additive:
// the string to mask by are added to the existing configuration.
Expand All @@ -260,9 +310,9 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont
//
// configuration = `['foo', 'bar']`
//
// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion
// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is
// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion
// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion
// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is
// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion
//
func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.Context {
lOpts := logging.GetProviderRootTFLoggerOpts(ctx)
Expand All @@ -271,3 +321,15 @@ func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.

return logging.SetProviderRootTFLoggerOpts(ctx, lOpts)
}

// MaskLogRegexes is a shortcut to invoke MaskMessageRegexes and MaskAllFieldValuesRegexes using the same input.
// Refer to those functions for details.
func MaskLogRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context {
return MaskMessageRegexes(MaskAllFieldValuesRegexes(ctx, expressions...), expressions...)
}

// MaskLogStrings is a shortcut to invoke MaskMessageStrings and MaskAllFieldValuesStrings using the same input.
// Refer to those functions for details.
func MaskLogStrings(ctx context.Context, matchingStrings ...string) context.Context {
return MaskMessageStrings(MaskAllFieldValuesStrings(ctx, matchingStrings...), matchingStrings...)
}