From 4409210b87f3d02640221fa958e46c17cb3193c2 Mon Sep 17 00:00:00 2001 From: Miccah Date: Mon, 21 Nov 2022 15:10:38 -0600 Subject: [PATCH] Add custom detectors configuration parsing (#927) * Add custom_detectors proto * Generate proto code * Create custom_detectors package Also create protoyaml package to test YAML unmarshalling the configuration. * Simplify custom_detectors proto by removing connection * Generate proto code * Update custom_detectors parsing tests --- go.mod | 2 + go.sum | 2 + pkg/custom_detectors/custom_detectors_test.go | 68 +++ .../custom_detectorspb/custom_detectors.pb.go | 348 +++++++++++++++ .../custom_detectors.pb.validate.go | 421 ++++++++++++++++++ pkg/protoyaml/protoyaml.go | 39 ++ proto/custom_detectors.proto | 25 ++ scripts/gen_proto.sh | 7 + 8 files changed, 912 insertions(+) create mode 100644 pkg/custom_detectors/custom_detectors_test.go create mode 100644 pkg/pb/custom_detectorspb/custom_detectors.pb.go create mode 100644 pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go create mode 100644 pkg/protoyaml/protoyaml.go create mode 100644 proto/custom_detectors.proto diff --git a/go.mod b/go.mod index 1563caeed4df..74dbbf161f3f 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( google.golang.org/protobuf v1.28.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/h2non/gock.v1 v1.1.2 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -143,5 +144,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.50.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index feccf6a3adc5..451133881be8 100644 --- a/go.sum +++ b/go.sum @@ -861,3 +861,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go new file mode 100644 index 000000000000..96a16001999d --- /dev/null +++ b/pkg/custom_detectors/custom_detectors_test.go @@ -0,0 +1,68 @@ +package custom_detectors + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml" +) + +const testCustomRegexYaml = `name: Internal bi tool +keywords: +- secret_v1_ +- pat_v2_ +regex: + id_pat_example: ([a-zA-Z0-9]{32}) + secret_pat_example: ([a-zA-Z0-9]{32}) +verify: +- endpoint: http://localhost:8000/{id_pat_example} + unsafe: true + headers: + - 'Authorization: Bearer {secret_pat_example.0}' + successRanges: + - 200-250 + - '288'` + +// Helper function to test equality to the data in testCustomRegexYaml. +func assertExpected(t *testing.T, got *custom_detectorspb.CustomRegex) { + assert.Equal(t, "Internal bi tool", got.Name) + assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords) + assert.Equal(t, map[string]string{ + "id_pat_example": "([a-zA-Z0-9]{32})", + "secret_pat_example": "([a-zA-Z0-9]{32})", + }, got.Regex) + assert.Equal(t, 1, len(got.Verify)) + assert.Equal(t, "http://localhost:8000/{id_pat_example}", got.Verify[0].Endpoint) + assert.Equal(t, true, got.Verify[0].Unsafe) + assert.Equal(t, []string{"Authorization: Bearer {secret_pat_example.0}"}, got.Verify[0].Headers) + assert.Equal(t, []string{"200-250", "288"}, got.Verify[0].SuccessRanges) +} + +func TestCustomRegexParsing(t *testing.T) { + var message custom_detectorspb.CustomRegex + + assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testCustomRegexYaml), &message)) + assertExpected(t, &message) +} + +func TestCustomDetectorsParsing(t *testing.T) { + var testYamlConfig string + // Build a config file using testCustomRegexYaml. + { + var lines []string + for i, line := range strings.Split(testCustomRegexYaml, "\n") { + if i == 0 { + lines = append(lines, line) + continue + } + lines = append(lines, " "+line) + } + testYamlConfig = "detectors:\n- " + strings.Join(lines, "\n") + } + + var messages custom_detectorspb.CustomDetectors + assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testYamlConfig), &messages)) + assertExpected(t, messages.Detectors[0]) +} diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.go new file mode 100644 index 000000000000..1d0a83fa3ed1 --- /dev/null +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.go @@ -0,0 +1,348 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.20.0 +// source: custom_detectors.proto + +package custom_detectorspb + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CustomDetectors struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Detectors []*CustomRegex `protobuf:"bytes,1,rep,name=detectors,proto3" json:"detectors,omitempty"` +} + +func (x *CustomDetectors) Reset() { + *x = CustomDetectors{} + if protoimpl.UnsafeEnabled { + mi := &file_custom_detectors_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CustomDetectors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CustomDetectors) ProtoMessage() {} + +func (x *CustomDetectors) ProtoReflect() protoreflect.Message { + mi := &file_custom_detectors_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CustomDetectors.ProtoReflect.Descriptor instead. +func (*CustomDetectors) Descriptor() ([]byte, []int) { + return file_custom_detectors_proto_rawDescGZIP(), []int{0} +} + +func (x *CustomDetectors) GetDetectors() []*CustomRegex { + if x != nil { + return x.Detectors + } + return nil +} + +type CustomRegex struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Keywords []string `protobuf:"bytes,2,rep,name=keywords,proto3" json:"keywords,omitempty"` + Regex map[string]string `protobuf:"bytes,3,rep,name=regex,proto3" json:"regex,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Verify []*VerifierConfig `protobuf:"bytes,4,rep,name=verify,proto3" json:"verify,omitempty"` +} + +func (x *CustomRegex) Reset() { + *x = CustomRegex{} + if protoimpl.UnsafeEnabled { + mi := &file_custom_detectors_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CustomRegex) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CustomRegex) ProtoMessage() {} + +func (x *CustomRegex) ProtoReflect() protoreflect.Message { + mi := &file_custom_detectors_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CustomRegex.ProtoReflect.Descriptor instead. +func (*CustomRegex) Descriptor() ([]byte, []int) { + return file_custom_detectors_proto_rawDescGZIP(), []int{1} +} + +func (x *CustomRegex) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CustomRegex) GetKeywords() []string { + if x != nil { + return x.Keywords + } + return nil +} + +func (x *CustomRegex) GetRegex() map[string]string { + if x != nil { + return x.Regex + } + return nil +} + +func (x *CustomRegex) GetVerify() []*VerifierConfig { + if x != nil { + return x.Verify + } + return nil +} + +type VerifierConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + Unsafe bool `protobuf:"varint,2,opt,name=unsafe,proto3" json:"unsafe,omitempty"` + Headers []string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty"` + SuccessRanges []string `protobuf:"bytes,4,rep,name=successRanges,proto3" json:"successRanges,omitempty"` +} + +func (x *VerifierConfig) Reset() { + *x = VerifierConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_custom_detectors_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *VerifierConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VerifierConfig) ProtoMessage() {} + +func (x *VerifierConfig) ProtoReflect() protoreflect.Message { + mi := &file_custom_detectors_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VerifierConfig.ProtoReflect.Descriptor instead. +func (*VerifierConfig) Descriptor() ([]byte, []int) { + return file_custom_detectors_proto_rawDescGZIP(), []int{2} +} + +func (x *VerifierConfig) GetEndpoint() string { + if x != nil { + return x.Endpoint + } + return "" +} + +func (x *VerifierConfig) GetUnsafe() bool { + if x != nil { + return x.Unsafe + } + return false +} + +func (x *VerifierConfig) GetHeaders() []string { + if x != nil { + return x.Headers + } + return nil +} + +func (x *VerifierConfig) GetSuccessRanges() []string { + if x != nil { + return x.SuccessRanges + } + return nil +} + +var File_custom_detectors_proto protoreflect.FileDescriptor + +var file_custom_detectors_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x4e, 0x0a, 0x0f, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x44, 0x65, 0x74, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74, + 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x73, 0x22, 0xf1, 0x01, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, + 0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, + 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, + 0x72, 0x64, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, + 0x78, 0x2e, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x65, + 0x67, 0x65, 0x78, 0x12, 0x38, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x1a, 0x38, 0x0a, + 0x0a, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, + 0x66, 0x69, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, + 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, + 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, + 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, + 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_custom_detectors_proto_rawDescOnce sync.Once + file_custom_detectors_proto_rawDescData = file_custom_detectors_proto_rawDesc +) + +func file_custom_detectors_proto_rawDescGZIP() []byte { + file_custom_detectors_proto_rawDescOnce.Do(func() { + file_custom_detectors_proto_rawDescData = protoimpl.X.CompressGZIP(file_custom_detectors_proto_rawDescData) + }) + return file_custom_detectors_proto_rawDescData +} + +var file_custom_detectors_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_custom_detectors_proto_goTypes = []interface{}{ + (*CustomDetectors)(nil), // 0: custom_detectors.CustomDetectors + (*CustomRegex)(nil), // 1: custom_detectors.CustomRegex + (*VerifierConfig)(nil), // 2: custom_detectors.VerifierConfig + nil, // 3: custom_detectors.CustomRegex.RegexEntry +} +var file_custom_detectors_proto_depIdxs = []int32{ + 1, // 0: custom_detectors.CustomDetectors.detectors:type_name -> custom_detectors.CustomRegex + 3, // 1: custom_detectors.CustomRegex.regex:type_name -> custom_detectors.CustomRegex.RegexEntry + 2, // 2: custom_detectors.CustomRegex.verify:type_name -> custom_detectors.VerifierConfig + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_custom_detectors_proto_init() } +func file_custom_detectors_proto_init() { + if File_custom_detectors_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_custom_detectors_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CustomDetectors); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_custom_detectors_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CustomRegex); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_custom_detectors_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VerifierConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_custom_detectors_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_custom_detectors_proto_goTypes, + DependencyIndexes: file_custom_detectors_proto_depIdxs, + MessageInfos: file_custom_detectors_proto_msgTypes, + }.Build() + File_custom_detectors_proto = out.File + file_custom_detectors_proto_rawDesc = nil + file_custom_detectors_proto_goTypes = nil + file_custom_detectors_proto_depIdxs = nil +} diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go new file mode 100644 index 000000000000..2b87120b3023 --- /dev/null +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go @@ -0,0 +1,421 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: custom_detectors.proto + +package custom_detectorspb + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on CustomDetectors with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *CustomDetectors) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on CustomDetectors with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// CustomDetectorsMultiError, or nil if none found. +func (m *CustomDetectors) ValidateAll() error { + return m.validate(true) +} + +func (m *CustomDetectors) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + for idx, item := range m.GetDetectors() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, CustomDetectorsValidationError{ + field: fmt.Sprintf("Detectors[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, CustomDetectorsValidationError{ + field: fmt.Sprintf("Detectors[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return CustomDetectorsValidationError{ + field: fmt.Sprintf("Detectors[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return CustomDetectorsMultiError(errors) + } + + return nil +} + +// CustomDetectorsMultiError is an error wrapping multiple validation errors +// returned by CustomDetectors.ValidateAll() if the designated constraints +// aren't met. +type CustomDetectorsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m CustomDetectorsMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m CustomDetectorsMultiError) AllErrors() []error { return m } + +// CustomDetectorsValidationError is the validation error returned by +// CustomDetectors.Validate if the designated constraints aren't met. +type CustomDetectorsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e CustomDetectorsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e CustomDetectorsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e CustomDetectorsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e CustomDetectorsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e CustomDetectorsValidationError) ErrorName() string { return "CustomDetectorsValidationError" } + +// Error satisfies the builtin error interface +func (e CustomDetectorsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sCustomDetectors.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = CustomDetectorsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = CustomDetectorsValidationError{} + +// Validate checks the field values on CustomRegex with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *CustomRegex) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on CustomRegex with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in CustomRegexMultiError, or +// nil if none found. +func (m *CustomRegex) ValidateAll() error { + return m.validate(true) +} + +func (m *CustomRegex) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Name + + // no validation rules for Regex + + for idx, item := range m.GetVerify() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, CustomRegexValidationError{ + field: fmt.Sprintf("Verify[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, CustomRegexValidationError{ + field: fmt.Sprintf("Verify[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return CustomRegexValidationError{ + field: fmt.Sprintf("Verify[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return CustomRegexMultiError(errors) + } + + return nil +} + +// CustomRegexMultiError is an error wrapping multiple validation errors +// returned by CustomRegex.ValidateAll() if the designated constraints aren't met. +type CustomRegexMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m CustomRegexMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m CustomRegexMultiError) AllErrors() []error { return m } + +// CustomRegexValidationError is the validation error returned by +// CustomRegex.Validate if the designated constraints aren't met. +type CustomRegexValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e CustomRegexValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e CustomRegexValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e CustomRegexValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e CustomRegexValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e CustomRegexValidationError) ErrorName() string { return "CustomRegexValidationError" } + +// Error satisfies the builtin error interface +func (e CustomRegexValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sCustomRegex.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = CustomRegexValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = CustomRegexValidationError{} + +// Validate checks the field values on VerifierConfig with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *VerifierConfig) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on VerifierConfig with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in VerifierConfigMultiError, +// or nil if none found. +func (m *VerifierConfig) ValidateAll() error { + return m.validate(true) +} + +func (m *VerifierConfig) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if _, err := url.Parse(m.GetEndpoint()); err != nil { + err = VerifierConfigValidationError{ + field: "Endpoint", + reason: "value must be a valid URI", + cause: err, + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Unsafe + + if len(errors) > 0 { + return VerifierConfigMultiError(errors) + } + + return nil +} + +// VerifierConfigMultiError is an error wrapping multiple validation errors +// returned by VerifierConfig.ValidateAll() if the designated constraints +// aren't met. +type VerifierConfigMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m VerifierConfigMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m VerifierConfigMultiError) AllErrors() []error { return m } + +// VerifierConfigValidationError is the validation error returned by +// VerifierConfig.Validate if the designated constraints aren't met. +type VerifierConfigValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e VerifierConfigValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e VerifierConfigValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e VerifierConfigValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e VerifierConfigValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e VerifierConfigValidationError) ErrorName() string { return "VerifierConfigValidationError" } + +// Error satisfies the builtin error interface +func (e VerifierConfigValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sVerifierConfig.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = VerifierConfigValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = VerifierConfigValidationError{} diff --git a/pkg/protoyaml/protoyaml.go b/pkg/protoyaml/protoyaml.go new file mode 100644 index 000000000000..7fe0c24bb6e5 --- /dev/null +++ b/pkg/protoyaml/protoyaml.go @@ -0,0 +1,39 @@ +package protoyaml + +import ( + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "sigs.k8s.io/yaml" +) + +var nonStrict = protojson.UnmarshalOptions{DiscardUnknown: true} +var strict = protojson.UnmarshalOptions{DiscardUnknown: false} + +// Marshal writes the given proto.Message in YAML format. +func Marshal(m proto.Message) ([]byte, error) { + json, err := protojson.Marshal(m) + if err != nil { + return nil, err + } + return yaml.JSONToYAML(json) +} + +// Unmarshal reads the given []byte into the given proto.Message, discarding +// any unknown fields in the input. +func Unmarshal(b []byte, m proto.Message) error { + json, err := yaml.YAMLToJSON(b) + if err != nil { + return err + } + return nonStrict.Unmarshal(json, m) +} + +// UnmarshalStrict reads the given []byte into the given proto.Message. If there +// are any unknown fields in the input, an error is returned. +func UnmarshalStrict(b []byte, m proto.Message) error { + json, err := yaml.YAMLToJSON(b) + if err != nil { + return err + } + return strict.Unmarshal(json, m) +} diff --git a/proto/custom_detectors.proto b/proto/custom_detectors.proto new file mode 100644 index 000000000000..04ef1ebb9ad2 --- /dev/null +++ b/proto/custom_detectors.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package custom_detectors; + +option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"; + +import "validate/validate.proto"; + +message CustomDetectors { + repeated CustomRegex detectors = 1; +} + +message CustomRegex { + string name = 1; + repeated string keywords = 2; + map regex = 3; + repeated VerifierConfig verify = 4; +} + +message VerifierConfig { + string endpoint = 1 [(validate.rules).string.uri_ref = true]; + bool unsafe = 2; + repeated string headers = 3; + repeated string successRanges = 4; +} diff --git a/scripts/gen_proto.sh b/scripts/gen_proto.sh index 226754cdf04b..d3df1c128fca 100755 --- a/scripts/gen_proto.sh +++ b/scripts/gen_proto.sh @@ -30,3 +30,10 @@ protoc -I proto/ \ --go_out=plugins=grpc:./pkg/pb/source_metadatapb --go_opt=paths=source_relative \ --validate_out="lang=go,paths=source_relative:./pkg/pb/source_metadatapb" \ proto/source_metadata.proto +protoc -I proto/ \ + -I ${GOPATH}/src \ + -I /usr/local/include \ + -I ${GOPATH}/src/github.com/envoyproxy/protoc-gen-validate \ + --go_out=plugins=grpc:./pkg/pb/custom_detectorspb --go_opt=paths=source_relative \ + --validate_out="lang=go,paths=source_relative:./pkg/pb/custom_detectorspb" \ + proto/custom_detectors.proto