From b3ff5d724111ab8719d5cbcaa90ffa360da5adc5 Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sun, 24 Jul 2022 10:11:04 +0100 Subject: [PATCH] Additional masking functions to be applied to log fields (#87) * Extending log filtering masking functionality to substrings inside field values * Wiring up the new functionalities to both the `tflog` and `tfsdklog` packages * Additional website doc * Changelog entry * Fix godoc --- .changelog/87.txt | 11 + internal/logging/filtering.go | 37 ++- internal/logging/filtering_test.go | 44 +++ internal/logging/options.go | 59 +++- tflog/provider.go | 88 +++++- tflog/provider_example_test.go | 96 +++++++ tflog/provider_test.go | 368 +++++++++++++++++++++++++ tflog/subsystem.go | 88 +++++- tflog/subsystem_example_test.go | 100 +++++++ tflog/subsystem_test.go | 372 ++++++++++++++++++++++++++ tfsdklog/sdk.go | 88 +++++- tfsdklog/sdk_example_test.go | 96 +++++++ tfsdklog/sdk_test.go | 368 +++++++++++++++++++++++++ tfsdklog/subsystem.go | 84 +++++- tfsdklog/subsystem_example_test.go | 100 +++++++ tfsdklog/subsystem_test.go | 372 ++++++++++++++++++++++++++ website/docs/plugin/log/filtering.mdx | 54 ++++ 17 files changed, 2362 insertions(+), 63 deletions(-) create mode 100644 .changelog/87.txt diff --git a/.changelog/87.txt b/.changelog/87.txt new file mode 100644 index 0000000..81ae9a1 --- /dev/null +++ b/.changelog/87.txt @@ -0,0 +1,11 @@ +```release-note:feature +tflog: Added `MaskAllFieldValuesRegexes()`, `MaskAllFieldValuesStrings()`, `MaskLogRegexes()` and `MaskLogStrings()` functions, which extend further the log masking filtering, for the provider root logger +``` + +```release-note:feature +tflog: Added `SubsystemMaskAllFieldValuesRegexes()`, `SubsystemMaskAllFieldValuesStrings()`, `SubsystemMaskLogRegexes()` and `SubsystemMaskLogStrings()` functions, which extend further the log masking filtering, for provider subsystem loggers +``` + +```release-note:feature +tfsdklog: Same functions added to the `tflog` package +``` diff --git a/internal/logging/filtering.go b/internal/logging/filtering.go index 9e7867f..c7b9c45 100644 --- a/internal/logging/filtering.go +++ b/internal/logging/filtering.go @@ -48,6 +48,7 @@ func (lo LoggerOpts) ShouldOmit(msg *string, fieldMaps ...map[string]interface{} // // Note that the given input is changed-in-place by this method. func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{}) { + // Replace any log field value with the corresponding field key equal to the configured strings if len(lo.MaskFieldValuesWithFieldKeys) > 0 { for _, k := range lo.MaskFieldValuesWithFieldKeys { for _, f := range fieldMaps { @@ -60,16 +61,44 @@ func (lo LoggerOpts) ApplyMask(msg *string, fieldMaps ...map[string]interface{}) } } - // Replace any part of the log message matching any of the configured regexp, - // with a masking replacement string + // Replace any part of any log field matching any of the configured regexp + if len(lo.MaskAllFieldValuesRegexes) > 0 { + for _, r := range lo.MaskAllFieldValuesRegexes { + for _, f := range fieldMaps { + for fk, fv := range f { + // Can apply the regexp replacement, only if the field value is indeed a string + fvStr, ok := fv.(string) + if ok { + f[fk] = r.ReplaceAllString(fvStr, logMaskingReplacementString) + } + } + } + } + } + + // Replace any part of any log field matching any of the configured strings + if len(lo.MaskAllFieldValuesStrings) > 0 { + for _, s := range lo.MaskAllFieldValuesStrings { + for _, f := range fieldMaps { + for fk, fv := range f { + // Can apply the regexp replacement, only if the field value is indeed a string + fvStr, ok := fv.(string) + if ok { + f[fk] = strings.ReplaceAll(fvStr, s, logMaskingReplacementString) + } + } + } + } + } + + // Replace any part of the log message matching any of the configured regexp if len(lo.MaskMessageRegexes) > 0 { for _, r := range lo.MaskMessageRegexes { *msg = r.ReplaceAllString(*msg, logMaskingReplacementString) } } - // Replace any part of the log message equal to any of the configured strings, - // with a masking replacement string + // Replace any part of the log message equal to any of the configured strings if len(lo.MaskMessageStrings) > 0 { for _, s := range lo.MaskMessageStrings { *msg = strings.ReplaceAll(*msg, s, logMaskingReplacementString) diff --git a/internal/logging/filtering_test.go b/internal/logging/filtering_test.go index 9929b09..ba50522 100644 --- a/internal/logging/filtering_test.go +++ b/internal/logging/filtering_test.go @@ -215,6 +215,50 @@ func TestApplyMask(t *testing.T) { }, }, }, + "mask-log-and-fields-matching-regexp": { + lOpts: logging.LoggerOpts{ + MaskMessageRegexes: []*regexp.Regexp{ + regexp.MustCompile("incorrectly configured BAZ"), + }, + MaskAllFieldValuesRegexes: []*regexp.Regexp{ + regexp.MustCompile("v1|v2"), + }, + }, + msg: testLogMsg, + fieldMaps: []map[string]interface{}{ + { + "k1": "v1 with some extra text", + "k2": "v2 with more extra text", + }, + }, + expectedMsg: "System FOO has caused error BAR because of ***", + expectedFieldMaps: []map[string]interface{}{ + { + "k1": "*** with some extra text", + "k2": "*** with more extra text", + }, + }, + }, + "mask-log-and-fields-matching-strings": { + lOpts: logging.LoggerOpts{ + MaskMessageStrings: []string{"FOO", "BAR", "BAZ"}, + MaskAllFieldValuesStrings: []string{"v1", "v2"}, + }, + msg: testLogMsg, + fieldMaps: []map[string]interface{}{ + { + "k1": "v1 with some extra text", + "k2": "v2 with more extra text", + }, + }, + expectedMsg: "System *** has caused error *** because of incorrectly configured ***", + expectedFieldMaps: []map[string]interface{}{ + { + "k1": "*** with some extra text", + "k2": "*** with more extra text", + }, + }, + }, "mask-log-by-key-and-matching-regexp": { lOpts: logging.LoggerOpts{ MaskMessageRegexes: []*regexp.Regexp{regexp.MustCompile("incorrectly configured BAZ")}, diff --git a/internal/logging/options.go b/internal/logging/options.go index 1f6ef03..8040699 100644 --- a/internal/logging/options.go +++ b/internal/logging/options.go @@ -56,9 +56,9 @@ type LoggerOpts struct { // // OmitLogWithFieldKeys = `['foo', 'baz']` // - // log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted - // log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed - // log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted + // log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted + // log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed + // log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted // OmitLogWithFieldKeys []string @@ -95,12 +95,41 @@ type LoggerOpts struct { // // MaskFieldValuesWithFieldKeys = `['foo', 'baz']` // - // log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value - // log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value - // log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value + // log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value + // log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value + // log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value // MaskFieldValuesWithFieldKeys []string + // MaskAllFieldValuesRegexes indicates that the logger should replace, within + // all the log field values, the portion matching one of the given *regexp.Regexp. + // + // Note that the replacement will happen, only for field values that are of type string. + // + // Example: + // + // MaskAllFieldValuesRegexes = `[regexp.MustCompile("(foo|bar)")]` + // + // log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value + // log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value + // log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value + // + MaskAllFieldValuesRegexes []*regexp.Regexp + + // MaskAllFieldValuesStrings indicates that the logger should replace, within + // all the log field values, the portion equal to one of the given strings. + // + // Note that the replacement will happen, only for field values that are of type string. + // + // Example: + // + // MaskAllFieldValuesStrings = `['foo', 'baz']` + // + // log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': 'bar', 'k3': '***' }` -> masked value + // log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': '***' }` -> as-is value + // log2 = `{ msg = "...", fields = { 'k1': '*** bar ***' }` -> masked value + MaskAllFieldValuesStrings []string + // MaskMessageRegexes indicates that the logger should replace, within // a log message, the portion matching one of the given *regexp.Regexp. // @@ -115,7 +144,7 @@ type LoggerOpts struct { MaskMessageRegexes []*regexp.Regexp // MaskMessageStrings indicates that the logger should replace, within - // a log message, the portion matching one of the given strings. + // a log message, the portion equal to one of the given strings. // // Example: // @@ -266,6 +295,22 @@ func WithMaskFieldValuesWithFieldKeys(keys ...string) Option { } } +// WithMaskAllFieldValuesRegexes appends keys to the LoggerOpts.MaskAllFieldValuesRegexes field. +func WithMaskAllFieldValuesRegexes(expressions ...*regexp.Regexp) Option { + return func(l LoggerOpts) LoggerOpts { + l.MaskAllFieldValuesRegexes = append(l.MaskAllFieldValuesRegexes, expressions...) + return l + } +} + +// WithMaskAllFieldValuesStrings appends keys to the LoggerOpts.MaskAllFieldValuesStrings field. +func WithMaskAllFieldValuesStrings(matchingStrings ...string) Option { + return func(l LoggerOpts) LoggerOpts { + l.MaskAllFieldValuesStrings = append(l.MaskAllFieldValuesStrings, matchingStrings...) + return l + } +} + // WithMaskMessageRegexes appends *regexp.Regexp to the LoggerOpts.MaskMessageRegexes field. func WithMaskMessageRegexes(expressions ...*regexp.Regexp) Option { return func(l LoggerOpts) LoggerOpts { diff --git a/tflog/provider.go b/tflog/provider.go index 0be81b5..bc3897e 100644 --- a/tflog/provider.go +++ b/tflog/provider.go @@ -146,9 +146,9 @@ func Error(ctx context.Context, msg string, additionalFields ...map[string]inter // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed -// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted +// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed +// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted // func OmitLogWithFieldKeys(ctx context.Context, keys ...string) context.Context { lOpts := logging.GetProviderRootTFLoggerOpts(ctx) @@ -214,9 +214,9 @@ func OmitLogWithMessageStrings(ctx context.Context, matchingStrings ...string) c // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value -// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value +// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value +// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value // func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.Context { lOpts := logging.GetProviderRootTFLoggerOpts(ctx) @@ -226,9 +226,59 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C return logging.SetProviderRootTFLoggerOpts(ctx, lOpts) } +// MaskAllFieldValuesRegexes returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// matching one of the given *regexp.Regexp. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func MaskAllFieldValuesRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context { + lOpts := logging.GetProviderRootTFLoggerOpts(ctx) + + lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts) + + return logging.SetProviderRootTFLoggerOpts(ctx, lOpts) +} + +// MaskAllFieldValuesStrings returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// equal to one of the given strings. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func MaskAllFieldValuesStrings(ctx context.Context, matchingStrings ...string) context.Context { + lOpts := logging.GetProviderRootTFLoggerOpts(ctx) + + lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts) + + return logging.SetProviderRootTFLoggerOpts(ctx, lOpts) +} + // MaskMessageRegexes returns a new context.Context that has a modified logger -// that masks (replaces) with asterisks (`***`) all message substrings matching one -// of the given strings. +// that masks (replaces) with asterisks (`***`) all message substrings, +// matching one of the given *regexp.Regexp. // // Each call to this function is additive: // the regexp to mask by are added to the existing configuration. @@ -250,8 +300,8 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont } // MaskMessageStrings returns a new context.Context that has a modified logger -// that masks (replace) with asterisks (`***`) all message substrings equal to one -// of the given strings. +// that masks (replace) with asterisks (`***`) all message substrings, +// equal to one of the given strings. // // Each call to this function is additive: // the string to mask by are added to the existing configuration. @@ -260,9 +310,9 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont // // configuration = `['foo', 'bar']` // -// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion -// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is -// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion +// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion +// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is +// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion // func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.Context { lOpts := logging.GetProviderRootTFLoggerOpts(ctx) @@ -271,3 +321,15 @@ func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context. return logging.SetProviderRootTFLoggerOpts(ctx, lOpts) } + +// MaskLogRegexes is a shortcut to invoke MaskMessageRegexes and MaskAllFieldValuesRegexes using the same input. +// Refer to those functions for details. +func MaskLogRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context { + return MaskMessageRegexes(MaskAllFieldValuesRegexes(ctx, expressions...), expressions...) +} + +// MaskLogStrings is a shortcut to invoke MaskMessageStrings and MaskAllFieldValuesStrings using the same input. +// Refer to those functions for details. +func MaskLogStrings(ctx context.Context, matchingStrings ...string) context.Context { + return MaskMessageStrings(MaskAllFieldValuesStrings(ctx, matchingStrings...), matchingStrings...) +} diff --git a/tflog/provider_example_test.go b/tflog/provider_example_test.go index 7b95ed8..f411f33 100644 --- a/tflog/provider_example_test.go +++ b/tflog/provider_example_test.go @@ -160,6 +160,54 @@ func ExampleMaskFieldValuesWithFieldKeys() { // {"@level":"trace","@message":"example log message","@module":"provider","field1":"***","field2":456} } +func ExampleMaskAllFieldValuesRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskAllFieldValuesRegexes(exampleCtx, regexp.MustCompile("v1|v2")) + + // all messages logged with exampleCtx will now have field values matching + // the above regular expressions, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"provider","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleMaskAllFieldValuesStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskAllFieldValuesStrings(exampleCtx, "v1", "v2") + + // all messages logged with exampleCtx will now have field values equal + // the above strings, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"provider","k1":"*** plus some text","k2":"*** plus more text"} +} + func ExampleMaskMessageStrings() { // virtually no plugin developers will need to worry about // instantiating loggers, as the libraries they're using will take care @@ -266,3 +314,51 @@ func ExampleOmitLogWithMessageRegexes() { // Output: // } + +func ExampleMaskLogRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskLogRegexes(exampleCtx, regexp.MustCompile("v1|v2"), regexp.MustCompile("message")) + + // all messages logged with exampleCtx will now have message content + // and field values matching the above regular expressions, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"provider","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleMaskLogStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskLogStrings(exampleCtx, "v1", "v2", "message") + + // all messages logged with exampleCtx will now have message content + // and field values equal the above strings, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"provider","k1":"*** plus some text","k2":"*** plus more text"} +} diff --git a/tflog/provider_test.go b/tflog/provider_test.go index 89fb18a..d859806 100644 --- a/tflog/provider_test.go +++ b/tflog/provider_test.go @@ -909,6 +909,190 @@ func TestMaskFieldValuesWithFieldKeys(t *testing.T) { } } +func TestMaskAllFieldValuesRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.MaskAllFieldValuesRegexes(ctx, testCase.expressions...) + + tflog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestMaskAllFieldValuesStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.MaskAllFieldValuesStrings(ctx, testCase.matchingStrings...) + + tflog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + func TestMaskMessageRegexes(t *testing.T) { testCases := map[string]struct { msg string @@ -1092,3 +1276,187 @@ func TestMaskMessageStrings(t *testing.T) { }) } } + +func TestMaskLogRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4"), regexp.MustCompile("(?i)BaAnAnA")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2"), regexp.MustCompile("FOO|BAR|BAZ")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "provider", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.MaskLogRegexes(ctx, testCase.expressions...) + + tflog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestMaskLogStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2", "FOO", "BAR", "BAZ"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "provider", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.MaskLogStrings(ctx, testCase.matchingStrings...) + + tflog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} diff --git a/tflog/subsystem.go b/tflog/subsystem.go index b63ed36..b491958 100644 --- a/tflog/subsystem.go +++ b/tflog/subsystem.go @@ -225,9 +225,9 @@ func SubsystemError(ctx context.Context, subsystem, msg string, additionalFields // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed -// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted +// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed +// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted // func SubsystemOmitLogWithFieldKeys(ctx context.Context, subsystem string, keys ...string) context.Context { lOpts := logging.GetProviderSubsystemTFLoggerOpts(ctx, subsystem) @@ -293,9 +293,9 @@ func SubsystemOmitLogWithMessageStrings(ctx context.Context, subsystem string, m // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value -// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value +// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value +// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value // func SubsystemMaskFieldValuesWithFieldKeys(ctx context.Context, subsystem string, keys ...string) context.Context { lOpts := logging.GetProviderSubsystemTFLoggerOpts(ctx, subsystem) @@ -305,9 +305,59 @@ func SubsystemMaskFieldValuesWithFieldKeys(ctx context.Context, subsystem string return logging.SetProviderSubsystemTFLoggerOpts(ctx, subsystem, lOpts) } +// SubsystemMaskAllFieldValuesRegexes returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// matching one of the given *regexp.Regexp. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func SubsystemMaskAllFieldValuesRegexes(ctx context.Context, subsystem string, expressions ...*regexp.Regexp) context.Context { + lOpts := logging.GetProviderSubsystemTFLoggerOpts(ctx, subsystem) + + lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts) + + return logging.SetProviderSubsystemTFLoggerOpts(ctx, subsystem, lOpts) +} + +// SubsystemMaskAllFieldValuesStrings returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// equal to one of the given strings. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func SubsystemMaskAllFieldValuesStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { + lOpts := logging.GetProviderSubsystemTFLoggerOpts(ctx, subsystem) + + lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts) + + return logging.SetProviderSubsystemTFLoggerOpts(ctx, subsystem, lOpts) +} + // SubsystemMaskMessageRegexes returns a new context.Context that has a modified logger -// that masks (replaces) with asterisks (`***`) all message substrings matching one -// of the given strings. +// that masks (replaces) with asterisks (`***`) all message substrings, +// matching one of the given *regexp.Regexp. // // Each call to this function is additive: // the regexp to mask by are added to the existing configuration. @@ -329,8 +379,8 @@ func SubsystemMaskMessageRegexes(ctx context.Context, subsystem string, expressi } // SubsystemMaskMessageStrings returns a new context.Context that has a modified logger -// that masks (replace) with asterisks (`***`) all message substrings equal to one -// of the given strings. +// that masks (replace) with asterisks (`***`) all message substrings, +// equal to one of the given strings. // // Each call to this function is additive: // the string to mask by are added to the existing configuration. @@ -339,9 +389,9 @@ func SubsystemMaskMessageRegexes(ctx context.Context, subsystem string, expressi // // configuration = `['foo', 'bar']` // -// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion -// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is -// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion +// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion +// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is +// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion // func SubsystemMaskMessageStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { lOpts := logging.GetProviderSubsystemTFLoggerOpts(ctx, subsystem) @@ -350,3 +400,15 @@ func SubsystemMaskMessageStrings(ctx context.Context, subsystem string, matching return logging.SetProviderSubsystemTFLoggerOpts(ctx, subsystem, lOpts) } + +// SubsystemMaskLogRegexes is a shortcut to invoke SubsystemMaskMessageRegexes and SubsystemMaskAllFieldValuesRegexes using the same input. +// Refer to those functions for details. +func SubsystemMaskLogRegexes(ctx context.Context, subsystem string, expressions ...*regexp.Regexp) context.Context { + return SubsystemMaskMessageRegexes(SubsystemMaskAllFieldValuesRegexes(ctx, subsystem, expressions...), subsystem, expressions...) +} + +// SubsystemMaskLogStrings is a shortcut to invoke SubsystemMaskMessageStrings and SubsystemMaskAllFieldValuesStrings using the same input. +// Refer to those functions for details. +func SubsystemMaskLogStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { + return SubsystemMaskMessageStrings(SubsystemMaskAllFieldValuesStrings(ctx, subsystem, matchingStrings...), subsystem, matchingStrings...) +} diff --git a/tflog/subsystem_example_test.go b/tflog/subsystem_example_test.go index 86aafac..524f767 100644 --- a/tflog/subsystem_example_test.go +++ b/tflog/subsystem_example_test.go @@ -228,6 +228,56 @@ func ExampleSubsystemMaskFieldValuesWithFieldKeys() { // {"@level":"trace","@message":"example log message","@module":"provider.my-subsystem","field1":"***","field2":456} } +func ExampleSubsystemMaskAllFieldValuesRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskAllFieldValuesRegexes(exampleCtx, "my-subsystem", regexp.MustCompile("v1|v2")) + + // all messages logged with exampleCtx will now have field values matching + // the above regular expressions, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"provider.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleSubsystemMaskAllFieldValuesStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskAllFieldValuesStrings(exampleCtx, "my-subsystem", "v1", "v2") + + // all messages logged with exampleCtx will now have field values equal + // the above strings, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"provider.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + func ExampleSubsystemMaskMessageStrings() { // virtually no plugin developers will need to worry about // instantiating loggers, as the libraries they're using will take care @@ -339,3 +389,53 @@ func ExampleSubsystemOmitLogWithMessageRegexes() { // Output: // } + +func ExampleSubsystemMaskLogRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskLogRegexes(exampleCtx, "my-subsystem", regexp.MustCompile("v1|v2"), regexp.MustCompile("message")) + + // all messages logged with exampleCtx will now have message content + // and field values matching the above regular expressions, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"provider.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleSubsystemMaskLogStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskLogStrings(exampleCtx, "my-subsystem", "v1", "v2", "message") + + // all messages logged with exampleCtx will now have message content + // and field values equal the above strings, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"provider.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} diff --git a/tflog/subsystem_test.go b/tflog/subsystem_test.go index 6fe763d..5c798a1 100644 --- a/tflog/subsystem_test.go +++ b/tflog/subsystem_test.go @@ -922,6 +922,192 @@ func TestSubsystemMaskFieldValuesWithFieldKeys(t *testing.T) { } } +func TestSubsystemMaskAllFieldValuesRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.NewSubsystem(ctx, testSubsystem) + ctx = tflog.SubsystemMaskAllFieldValuesRegexes(ctx, testSubsystem, testCase.expressions...) + + tflog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestSubsystemMaskAllFieldValuesStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.NewSubsystem(ctx, testSubsystem) + ctx = tflog.SubsystemMaskAllFieldValuesStrings(ctx, testSubsystem, testCase.matchingStrings...) + + tflog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + func TestSubsystemMaskMessageRegexes(t *testing.T) { testCases := map[string]struct { msg string @@ -1107,3 +1293,189 @@ func TestSubsystemMaskMessageStrings(t *testing.T) { }) } } + +func TestSubsystemMaskLogRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4"), regexp.MustCompile("(?i)BaAnAnA")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2"), regexp.MustCompile("FOO|BAR|BAZ")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "provider.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.NewSubsystem(ctx, testSubsystem) + ctx = tflog.SubsystemMaskLogRegexes(ctx, testSubsystem, testCase.expressions...) + + tflog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestSubsystemMaskLogStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "provider.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2", "FOO", "BAR", "BAZ"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "provider.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.ProviderRoot(ctx, &outputBuffer) + ctx = tflog.NewSubsystem(ctx, testSubsystem) + ctx = tflog.SubsystemMaskLogStrings(ctx, testSubsystem, testCase.matchingStrings...) + + tflog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} diff --git a/tfsdklog/sdk.go b/tfsdklog/sdk.go index eca9727..3a74983 100644 --- a/tfsdklog/sdk.go +++ b/tfsdklog/sdk.go @@ -229,9 +229,9 @@ func Error(ctx context.Context, msg string, additionalFields ...map[string]inter // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed -// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted +// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed +// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted // func OmitLogWithFieldKeys(ctx context.Context, keys ...string) context.Context { lOpts := logging.GetSDKRootTFLoggerOpts(ctx) @@ -297,9 +297,9 @@ func OmitLogWithMessageStrings(ctx context.Context, matchingStrings ...string) c // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value -// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value +// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value +// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value // func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.Context { lOpts := logging.GetSDKRootTFLoggerOpts(ctx) @@ -309,9 +309,59 @@ func MaskFieldValuesWithFieldKeys(ctx context.Context, keys ...string) context.C return logging.SetSDKRootTFLoggerOpts(ctx, lOpts) } +// MaskAllFieldValuesRegexes returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// matching one of the given *regexp.Regexp. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func MaskAllFieldValuesRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context { + lOpts := logging.GetSDKRootTFLoggerOpts(ctx) + + lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts) + + return logging.SetSDKRootTFLoggerOpts(ctx, lOpts) +} + +// MaskAllFieldValuesStrings returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// equal to one of the given strings. +// +// Note that the replacement will happen, only for field values that are of type string. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func MaskAllFieldValuesStrings(ctx context.Context, matchingStrings ...string) context.Context { + lOpts := logging.GetSDKRootTFLoggerOpts(ctx) + + lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts) + + return logging.SetSDKRootTFLoggerOpts(ctx, lOpts) +} + // MaskMessageRegexes returns a new context.Context that has a modified logger -// that masks (replaces) with asterisks (`***`) all message substrings matching one -// of the given strings. +// that masks (replaces) with asterisks (`***`) all message substrings, +// matching one of the given *regexp.Regexp. // // Each call to this function is additive: // the regexp to mask by are added to the existing configuration. @@ -333,8 +383,8 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont } // MaskMessageStrings returns a new context.Context that has a modified logger -// that masks (replace) with asterisks (`***`) all message substrings equal to one -// of the given strings. +// that masks (replace) with asterisks (`***`) all message substrings, +// equal to one of the given strings. // // Each call to this function is additive: // the string to mask by are added to the existing configuration. @@ -343,9 +393,9 @@ func MaskMessageRegexes(ctx context.Context, expressions ...*regexp.Regexp) cont // // configuration = `['foo', 'bar']` // -// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion -// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is -// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion +// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion +// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is +// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion // func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context.Context { lOpts := logging.GetSDKRootTFLoggerOpts(ctx) @@ -354,3 +404,15 @@ func MaskMessageStrings(ctx context.Context, matchingStrings ...string) context. return logging.SetSDKRootTFLoggerOpts(ctx, lOpts) } + +// MaskLogRegexes is a shortcut to invoke MaskMessageRegexes and MaskAllFieldValuesRegexes using the same input. +// Refer to those functions for details. +func MaskLogRegexes(ctx context.Context, expressions ...*regexp.Regexp) context.Context { + return MaskMessageRegexes(MaskAllFieldValuesRegexes(ctx, expressions...), expressions...) +} + +// MaskLogStrings is a shortcut to invoke MaskMessageStrings and MaskAllFieldValuesStrings using the same input. +// Refer to those functions for details. +func MaskLogStrings(ctx context.Context, matchingStrings ...string) context.Context { + return MaskMessageStrings(MaskAllFieldValuesStrings(ctx, matchingStrings...), matchingStrings...) +} diff --git a/tfsdklog/sdk_example_test.go b/tfsdklog/sdk_example_test.go index 5b2affb..3914591 100644 --- a/tfsdklog/sdk_example_test.go +++ b/tfsdklog/sdk_example_test.go @@ -146,6 +146,54 @@ func ExampleMaskFieldValuesWithFieldKeys() { // {"@level":"trace","@message":"example log message","@module":"sdk","field1":"***","field2":456} } +func ExampleMaskAllFieldValuesRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskAllFieldValuesRegexes(exampleCtx, regexp.MustCompile("v1|v2")) + + // all messages logged with exampleCtx will now have field values matching + // the above regular expressions, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"sdk","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleMaskAllFieldValuesStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskAllFieldValuesStrings(exampleCtx, "v1", "v2") + + // all messages logged with exampleCtx will now have field values equal + // the above strings, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"sdk","k1":"*** plus some text","k2":"*** plus more text"} +} + func ExampleMaskMessageStrings() { // virtually no plugin developers will need to worry about // instantiating loggers, as the libraries they're using will take care @@ -252,3 +300,51 @@ func ExampleOmitLogWithMessageRegexes() { // Output: // } + +func ExampleMaskLogRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskLogRegexes(exampleCtx, regexp.MustCompile("v1|v2"), regexp.MustCompile("message")) + + // all messages logged with exampleCtx will now have message content + // and field values matching the above regular expressions, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"sdk","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleMaskLogStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = MaskLogStrings(exampleCtx, "v1", "v2", "message") + + // all messages logged with exampleCtx will now have message content + // and field values equal the above strings, replaced with *** + Trace(exampleCtx, "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"sdk","k1":"*** plus some text","k2":"*** plus more text"} +} diff --git a/tfsdklog/sdk_test.go b/tfsdklog/sdk_test.go index ca0cf8e..cd57f73 100644 --- a/tfsdklog/sdk_test.go +++ b/tfsdklog/sdk_test.go @@ -909,6 +909,190 @@ func TestMaskFieldValuesWithFieldKeys(t *testing.T) { } } +func TestMaskAllFieldValuesRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.MaskAllFieldValuesRegexes(ctx, testCase.expressions...) + + tfsdklog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestMaskAllFieldValuesStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.MaskAllFieldValuesStrings(ctx, testCase.matchingStrings...) + + tfsdklog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + func TestMaskMessageRegexes(t *testing.T) { testCases := map[string]struct { msg string @@ -1092,3 +1276,187 @@ func TestMaskMessageStrings(t *testing.T) { }) } } + +func TestMaskLogRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4"), regexp.MustCompile("(?i)BaAnAnA")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2"), regexp.MustCompile("FOO|BAR|BAZ")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "sdk", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.MaskLogRegexes(ctx, testCase.expressions...) + + tfsdklog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestMaskLogStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2", "FOO", "BAR", "BAZ"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "sdk", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.MaskLogStrings(ctx, testCase.matchingStrings...) + + tfsdklog.Trace(ctx, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} diff --git a/tfsdklog/subsystem.go b/tfsdklog/subsystem.go index bba1d6f..bfad8d9 100644 --- a/tfsdklog/subsystem.go +++ b/tfsdklog/subsystem.go @@ -225,9 +225,9 @@ func SubsystemError(ctx context.Context, subsystem, msg string, additionalFields // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '...', 'bar', '...' }` -> omitted -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> printed -// log3 = `{ msg = "...", fields = { 'baz`', '...', 'boo', '...' }` -> omitted +// log1 = `{ msg = "...", fields = { 'foo': '...', 'bar': '...' }` -> omitted +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> printed +// log3 = `{ msg = "...", fields = { 'baz': '...', 'boo': '...' }` -> omitted // func SubsystemOmitLogWithFieldKeys(ctx context.Context, subsystem string, keys ...string) context.Context { lOpts := logging.GetSDKSubsystemTFLoggerOpts(ctx, subsystem) @@ -293,9 +293,9 @@ func SubsystemOmitLogWithMessageStrings(ctx context.Context, subsystem string, m // // configuration = `['foo', 'baz']` // -// log1 = `{ msg = "...", fields = { 'foo', '***', 'bar', '...' }` -> masked value -// log2 = `{ msg = "...", fields = { 'bar', '...' }` -> as-is value -// log3 = `{ msg = "...", fields = { 'baz`', '***', 'boo', '...' }` -> masked value +// log1 = `{ msg = "...", fields = { 'foo': '***', 'bar': '...' }` -> masked value +// log2 = `{ msg = "...", fields = { 'bar': '...' }` -> as-is value +// log3 = `{ msg = "...", fields = { 'baz': '***', 'boo': '...' }` -> masked value // func SubsystemMaskFieldValuesWithFieldKeys(ctx context.Context, subsystem string, keys ...string) context.Context { lOpts := logging.GetSDKSubsystemTFLoggerOpts(ctx, subsystem) @@ -305,9 +305,55 @@ func SubsystemMaskFieldValuesWithFieldKeys(ctx context.Context, subsystem string return logging.SetSDKSubsystemTFLoggerOpts(ctx, subsystem, lOpts) } +// SubsystemMaskAllFieldValuesRegexes returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// matching one of the given *regexp.Regexp. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func SubsystemMaskAllFieldValuesRegexes(ctx context.Context, subsystem string, expressions ...*regexp.Regexp) context.Context { + lOpts := logging.GetSDKSubsystemTFLoggerOpts(ctx, subsystem) + + lOpts = logging.WithMaskAllFieldValuesRegexes(expressions...)(lOpts) + + return logging.SetSDKSubsystemTFLoggerOpts(ctx, subsystem, lOpts) +} + +// SubsystemMaskAllFieldValuesStrings returns a new context.Context that has a modified logger +// that masks (replaces) with asterisks (`***`) all field value substrings, +// equal to one of the given strings. +// +// Each call to this function is additive: +// the regexp to mask by are added to the existing configuration. +// +// Example: +// +// configuration = `[regexp.MustCompile("(foo|bar)")]` +// +// log1 = `{ msg = "...", fields = { 'k1': '***', 'k2': '***', 'k3': 'baz' }` -> masked value +// log2 = `{ msg = "...", fields = { 'k1': 'boo', 'k2': 'far', 'k3': 'baz' }` -> as-is value +// log2 = `{ msg = "...", fields = { 'k1': '*** *** baz' }` -> masked value +// +func SubsystemMaskAllFieldValuesStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { + lOpts := logging.GetSDKSubsystemTFLoggerOpts(ctx, subsystem) + + lOpts = logging.WithMaskAllFieldValuesStrings(matchingStrings...)(lOpts) + + return logging.SetSDKSubsystemTFLoggerOpts(ctx, subsystem, lOpts) +} + // SubsystemMaskMessageRegexes returns a new context.Context that has a modified logger -// that masks (replaces) with asterisks (`***`) all message substrings matching one -// of the given strings. +// that masks (replaces) with asterisks (`***`) all message substrings, +// matching one of the given *regexp.Regexp. // // Each call to this function is additive: // the regexp to mask by are added to the existing configuration. @@ -329,8 +375,8 @@ func SubsystemMaskMessageRegexes(ctx context.Context, subsystem string, expressi } // SubsystemMaskMessageStrings returns a new context.Context that has a modified logger -// that masks (replace) with asterisks (`***`) all message substrings equal to one -// of the given strings. +// that masks (replace) with asterisks (`***`) all message substrings, +// equal to one of the given strings. // // Each call to this function is additive: // the string to mask by are added to the existing configuration. @@ -339,9 +385,9 @@ func SubsystemMaskMessageRegexes(ctx context.Context, subsystem string, expressi // // configuration = `['foo', 'bar']` // -// log1 = `{ msg = "banana apple ***", fields = {...}` -> masked portion -// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is -// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion +// log1 = `{ msg = "banana apple ***", fields = { 'k1': 'foo, bar, baz' }` -> masked portion +// log2 = `{ msg = "pineapple mango", fields = {...}` -> as-is +// log3 = `{ msg = "pineapple mango ***", fields = {...}` -> masked portion // func SubsystemMaskMessageStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { lOpts := logging.GetSDKSubsystemTFLoggerOpts(ctx, subsystem) @@ -350,3 +396,15 @@ func SubsystemMaskMessageStrings(ctx context.Context, subsystem string, matching return logging.SetSDKSubsystemTFLoggerOpts(ctx, subsystem, lOpts) } + +// SubsystemMaskLogRegexes is a shortcut to invoke SubsystemMaskMessageRegexes and SubsystemMaskAllFieldValuesRegexes using the same input. +// Refer to those functions for details. +func SubsystemMaskLogRegexes(ctx context.Context, subsystem string, expressions ...*regexp.Regexp) context.Context { + return SubsystemMaskMessageRegexes(SubsystemMaskAllFieldValuesRegexes(ctx, subsystem, expressions...), subsystem, expressions...) +} + +// SubsystemMaskLogStrings is a shortcut to invoke SubsystemMaskMessageStrings and SubsystemMaskAllFieldValuesStrings using the same input. +// Refer to those functions for details. +func SubsystemMaskLogStrings(ctx context.Context, subsystem string, matchingStrings ...string) context.Context { + return SubsystemMaskMessageStrings(SubsystemMaskAllFieldValuesStrings(ctx, subsystem, matchingStrings...), subsystem, matchingStrings...) +} diff --git a/tfsdklog/subsystem_example_test.go b/tfsdklog/subsystem_example_test.go index 015feff..864ba3a 100644 --- a/tfsdklog/subsystem_example_test.go +++ b/tfsdklog/subsystem_example_test.go @@ -194,6 +194,56 @@ func ExampleSubsystemMaskFieldValuesWithFieldKeys() { // {"@level":"trace","@message":"example log message","@module":"sdk.my-subsystem","field1":"***","field2":456} } +func ExampleSubsystemMaskAllFieldValuesRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskAllFieldValuesRegexes(exampleCtx, "my-subsystem", regexp.MustCompile("v1|v2")) + + // all messages logged with exampleCtx will now have field values matching + // the above regular expressions, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"sdk.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleSubsystemMaskAllFieldValuesStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskAllFieldValuesStrings(exampleCtx, "my-subsystem", "v1", "v2") + + // all messages logged with exampleCtx will now have field values equal + // the above strings, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log message","@module":"sdk.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + func ExampleSubsystemMaskMessageStrings() { // virtually no plugin developers will need to worry about // instantiating loggers, as the libraries they're using will take care @@ -305,3 +355,53 @@ func ExampleSubsystemOmitLogWithMessageRegexes() { // Output: // } + +func ExampleSubsystemMaskLogRegexes() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskLogRegexes(exampleCtx, "my-subsystem", regexp.MustCompile("v1|v2"), regexp.MustCompile("message")) + + // all messages logged with exampleCtx will now have message content + // and field values matching the above regular expressions, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"sdk.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} + +func ExampleSubsystemMaskLogStrings() { + // virtually no plugin developers will need to worry about + // instantiating loggers, as the libraries they're using will take care + // of that, but we're not using those libraries in these examples. So + // we need to do the injection ourselves. Plugin developers will + // basically never need to do this, so the next line can safely be + // considered setup for the example and ignored. Instead, use the + // context passed in by the framework or library you're using. + exampleCtx := getExampleContext() + + // non-example-setup code begins here + exampleCtx = NewSubsystem(exampleCtx, "my-subsystem") + exampleCtx = SubsystemMaskLogStrings(exampleCtx, "my-subsystem", "v1", "v2", "message") + + // all messages logged with exampleCtx will now have message content + // and field values equal the above strings, replaced with *** + SubsystemTrace(exampleCtx, "my-subsystem", "example log message", map[string]interface{}{ + "k1": "v1 plus some text", + "k2": "v2 plus more text", + }) + + // Output: + // {"@level":"trace","@message":"example log ***","@module":"sdk.my-subsystem","k1":"*** plus some text","k2":"*** plus more text"} +} diff --git a/tfsdklog/subsystem_test.go b/tfsdklog/subsystem_test.go index c48c580..ff4f0eb 100644 --- a/tfsdklog/subsystem_test.go +++ b/tfsdklog/subsystem_test.go @@ -922,6 +922,192 @@ func TestSubsystemMaskFieldValuesWithFieldKeys(t *testing.T) { } } +func TestSubsystemMaskAllFieldValuesRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.NewSubsystem(ctx, testSubsystem) + ctx = tfsdklog.SubsystemMaskAllFieldValuesRegexes(ctx, testSubsystem, testCase.expressions...) + + tfsdklog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestSubsystemMaskAllFieldValuesStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.NewSubsystem(ctx, testSubsystem) + ctx = tfsdklog.SubsystemMaskAllFieldValuesStrings(ctx, testSubsystem, testCase.matchingStrings...) + + tfsdklog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + func TestSubsystemMaskMessageRegexes(t *testing.T) { testCases := map[string]struct { msg string @@ -1107,3 +1293,189 @@ func TestSubsystemMaskMessageStrings(t *testing.T) { }) } } + +func TestSubsystemMaskLogRegexes(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + expressions []*regexp.Regexp + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v3"), regexp.MustCompile("v4"), regexp.MustCompile("(?i)BaAnAnA")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-regexp": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + expressions: []*regexp.Regexp{regexp.MustCompile("v1|v2"), regexp.MustCompile("FOO|BAR|BAZ")}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "sdk.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.NewSubsystem(ctx, testSubsystem) + ctx = tfsdklog.SubsystemMaskLogRegexes(ctx, testSubsystem, testCase.expressions...) + + tfsdklog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} + +func TestSubsystemMaskLogStrings(t *testing.T) { + testCases := map[string]struct { + msg string + additionalFields []map[string]interface{} + matchingStrings []string + expectedOutput []map[string]interface{} + }{ + "no-masking": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "no-matches": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1", + "k2": "v2", + }, + }, + matchingStrings: []string{"v3", "v4"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System FOO has caused error BAR because of incorrectly configured BAZ", + "@module": "sdk.test_subsystem", + "k1": "v1", + "k2": "v2", + }, + }, + }, + "mask-matching-strings": { + msg: testLogMsg, + additionalFields: []map[string]interface{}{ + { + "k1": "v1 plus text", + "k2": "v2 more text", + }, + }, + matchingStrings: []string{"v1", "v2", "FOO", "BAR", "BAZ"}, + expectedOutput: []map[string]interface{}{ + { + "@level": "trace", + "@message": "System *** has caused error *** because of incorrectly configured ***", + "@module": "sdk.test_subsystem", + "k1": "*** plus text", + "k2": "*** more text", + }, + }, + }, + } + + for name, testCase := range testCases { + name, testCase := name, testCase + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var outputBuffer bytes.Buffer + + ctx := context.Background() + ctx = loggertest.SDKRoot(ctx, &outputBuffer) + ctx = tfsdklog.NewSubsystem(ctx, testSubsystem) + ctx = tfsdklog.SubsystemMaskLogStrings(ctx, testSubsystem, testCase.matchingStrings...) + + tfsdklog.SubsystemTrace(ctx, testSubsystem, testCase.msg, testCase.additionalFields...) + + got, err := loggertest.MultilineJSONDecode(&outputBuffer) + if err != nil { + t.Fatalf("unable to read multiple line JSON: %s", err) + } + + if diff := cmp.Diff(testCase.expectedOutput, got); diff != "" { + t.Errorf("unexpected output difference: %s", diff) + } + }) + } +} diff --git a/website/docs/plugin/log/filtering.mdx b/website/docs/plugin/log/filtering.mdx index 86bc9b6..9f60321 100644 --- a/website/docs/plugin/log/filtering.mdx +++ b/website/docs/plugin/log/filtering.mdx @@ -98,6 +98,60 @@ tflog.MaskFieldValuesWithFieldKeys(ctx, sensitiveFields...) tflog.MaskFieldValuesWithFieldKeys(ctx, "yet-another-sensitive-field", "final-sensitive-field") ``` +### Masking Field Values Via Regular Expressions + +Use the [`tflog.MaskAllFieldValuesRegexes()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#MaskAllFieldValuesRegexes) before writing logs: + +```go +tflog.MaskAllFieldValuesRegexes(ctx, regexp.MustCompile(`(\w{3}_SECRET)`)) + +// Will output: example message: contains-secret=my-super-*** +tflog.Trace(ctx, "example message", map[string]interface{}{"contains-secret": "my-super-TOP_SECRET"}) +``` + +For subsystems, use the [`tflog.SubsystemMaskAllFieldValuesRegexes()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#SubsystemMaskAllFieldValuesRegexes) before writing logs: + +```go +tflog.SubsystemMaskAllFieldValuesRegexes(ctx, "my-subsystem", regexp.MustCompile(`(\w{3}_SECRET)`)) +``` + +Both functions can accept multiple string values at once to simplify filtering implementations. + +### Masking Field Values Via Exact Strings + +Use the [`tflog.MaskAllFieldValuesStrings()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#MaskAllFieldValuesStrings) before writing logs: + +```go +tflog.MaskAllFieldValuesStrings(ctx, "TOP_SECRET") + +// Will output: example message: contains-secret=my-super-*** +tflog.Trace(ctx, "example message", map[string]interface{}{"contains-secret": "my-super-TOP_SECRET"}) +``` + +For subsystems, use the [`tflog.SubsystemMaskAllFieldValuesStrings()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#SubsystemMaskAllFieldValuesStrings) before writing logs: + +```go +tflog.SubsystemMaskAllFieldValuesStrings(ctx, "my-subsystem", "TOP_SECRET") +``` + +Both functions can accept multiple string values at once to simplify filtering implementations. + +### Masking Messages and Field Values via Regular Expressions + +Use the [`tflog.MaskLogRegexes()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#MaskLogRegexes) to +obtain the same configuration and behaviour as if you had used the same input on `tflog.MaskMessageRegexes()` and `tflog.MaskAllFieldValuesRegexes()`. + +The same applies, respectively, for [`tflog.SubsystemMaskLogRegexes()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#SubsystemMaskLogRegexes), +and the functions `tflog.SubsystemMaskMessageRegexes()` and `tflog.SubsystemMaskAllFieldValuesRegexes()`. + +### Masking Messages and Field Values via Exact Strings + +Use the [`tflog.MaskLogStrings()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#MaskLogStrings) to +obtain the same configuration and behaviour as if you had used the same input on `tflog.MaskMessageStrings()` and `tflog.MaskAllFieldValuesStrings()`. + +The same applies, respectively, for [`tflog.SubsystemMaskLogStrings()` function](https://pkg.go.dev/github.com/hashicorp/terraform-plugin-log/tflog#SubsystemMaskLogStrings), +and the functions `tflog.SubsystemMaskMessageStrings()` and `tflog.SubsystemMaskAllFieldValuesStrings()`. + ## Omitting Log Output Omitting logs is the process of skipping the entire output of a log message and its fields when encountering sensitive data.