feat: Refactor Inbound package to provide access to SendGrid's pre-pr…

…ocessing (#443)

* primary code transfer and expansion

* add validation method

* pass existing tests

* add a test for validation

* add parsed fields and update docs

* parse envelope

* update docs

* docs

* whitespace

* simplify

* comments

* defensively checks parsed header to avoid breaking email clients

* updates warning in documentation

* updates docs, DRY, and adds an additional test

helpers/inbound/README.md

@@ -2,10 +2,39 @@
 
 ## Table of Contents
 
+* [Fields](#fields)
 * [Example Usage](#example-usage)
 * [Testing the Source Code](#testing)
 * [Contributing](#contributing)
 
+# Fields
+
+### ParsedEmail.Envelope
+  ParsedEmail.Envelope.To and ParsedEmail.Envelope.From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses
+
+### ParsedEmail.ParsedValues
+  Please see [SendGrid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent
+
+### ParsedEmail.TextBody
+  this field will satisfy  most cases. SendGrid pre-parses the body into a plain text string separated with \n
+
+### ParsedEmail.ParsedAttachments
+  populated **only** when processing the email with ParseWithAttachments(). Provides the following ease of use values
+  - File: full attachment for uploading or processing (see example to upload to s3)
+  - Size: file size, useful for filtering or setting upper limits to attachments
+  - Filename: copies the original filename of the attachment, if there is not one, it defaults to 'Untitled'
+  - ContentType: the type of file
+
+### ParsedEmail.Body
+  populated *only* when the raw option is checked in the SendGrid Dashboard. Provides the raw HTML body of the email, unless you need to record the exact unparsed HTML payload from the email client, you should use the parsed fields instead. The field is named Body for backward compatability
+
+### ParsedEmail.Attachments
+  populated *only* when the raw option is checked in the SendGrid Dashboard. This field is deprecated. Use ParsedAttachments instead which does not require the Raw setting, and provides parsed values to use and process the attachments
+
+### ParsedEmail.Headers
+  this field is deprecated. Use the SendGrid processed fields in ParsedValues instead. While it maintains its presence to avoid breaking changes, it provides raw, unprocessed headers and not all email clients are compatible. For example. these fields will be empty if the email cient is Outlook.com
+
+
 # Example Usage
 
 ```go
@@ -20,27 +49,40 @@ import (
 )
 
 func inboundHandler(response http.ResponseWriter, request *http.Request) {
-	parsedEmail, err := Parse(request)
+	parsedEmail, err := ParseWithAttachments(request)
 	if err != nil {
 		log.Fatal(err)
 	}
-    
-	fmt.Print(parsedEmail.Headers["From"])
-	
-	for filename, contents := range parsedEmail.Attachments {
+
+	fmt.Print(parsedEmail.Envelope.From)
+
+	for filename, contents := range parsedEmail.ParsedAttachments {
 		// Do something with an attachment
 		handleAttachment(filename, contents)
 	}
-    
-	for section, body := range parsedEmail.Body {
-		// Do something with the email body
-		handleEmail(body)
+
+
+	for section, body := range strings.Split(parsedEmail.TextBody, "\n") {
+		// Do something with the email lines
 	}
-
+
+
 	// Twilio SendGrid needs a 200 OK response to stop POSTing
 	response.WriteHeader(http.StatusOK)
 }
 
+// example of uploading an attachment to s3 using the Go sdk-2
+func handleAttachment(parsedEmail *ParsedEmail) {
+	for _, contents := range parsedEmail.ParsedAttachments {
+			if _, err := sgh.Client.Upload(ctx, &s3.PutObjectInput{
+				Bucket:               &bucket,
+				Key:                  &uploadPath,
+				Body:                 contents.File,
+				ContentType:          aws.String(contents.ContentType),
+      }
+  }
+}
+
 func main() {
 	http.HandleFunc("/inbound", inboundHandler)
 	if err := http.ListenAndServe(":8000", nil); err != nil {

helpers/inbound/inbound.go

@@ -1,6 +1,8 @@
 package inbound
 
 import (
+	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -9,20 +11,72 @@ import (
 	"strings"
 )
 
+// ParsedEmail defines a multipart parsed email
+// Body and Attachments are only populated if the Raw option is checked on the SendGrid inbound configuration and are named for backwards compatability
 type ParsedEmail struct {
-	Headers     map[string]string
-	Body        map[string]string
+	// Header values are raw and not pre-processed by SendGrid. They may change depending on the email client. Use carefully
+	Headers map[string]string
+	// Please see https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook to see the available fields in the email headers
+	// all fields listed there are available within the headers map except for text which lives in the TextBody field
+	ParsedValues map[string]string
+	// Primary email body parsed with \n. A common approach is to Split by the \n to bring every line of the email into a string array
+	TextBody string
+
+	// Envelope expresses the exact email address that the email was addressed to and the exact email address it was from, without extra characters
+	Envelope struct {
+		From string   `json:"from"`
+		To   []string `json:"to"`
+	}
+
+	// Attachments have been fully parsed to include the filename, size, content type and actual file for uploading or processing
+	ParsedAttachments map[string]*EmailAttachment
+
+	// Raw only
 	Attachments map[string][]byte
-	rawRequest  *http.Request
+	// accessed with text/html and text/plain. text/plain is always parsed to the TextBody field
+	Body map[string]string
+
+	rawRequest      *http.Request
+	rawValues       map[string][]string
+	withAttachments bool
 }
 
-func Parse(request *http.Request) (*ParsedEmail, error) {
-	result := ParsedEmail{
-		Headers:     make(map[string]string),
+// EmailAttachment defines information related to an email attachment
+type EmailAttachment struct {
+	File        multipart.File `json:"-"`
+	Filename    string         `json:"filename"`
+	Size        int64          `json:"-"`
+	ContentType string         `json:"type"`
+}
+
+func newParsedEmail(request *http.Request) ParsedEmail {
+	return ParsedEmail{
+		Headers:           make(map[string]string),
+		ParsedValues:      make(map[string]string),
+		ParsedAttachments: make(map[string]*EmailAttachment),
+
 		Body:        make(map[string]string),
 		Attachments: make(map[string][]byte),
-		rawRequest:  request,
+
+		rawRequest:      request,
+		withAttachments: false,
 	}
+}
+
+// Parse parses an email using Go's multipart parser and populates the headers, and body
+// This method skips processing the attachment file and is therefore more performant
+func Parse(request *http.Request) (*ParsedEmail, error) {
+	result := newParsedEmail(request)
+
+	err := result.parse()
+	return &result, err
+}
+
+// ParseWithAttachments parses an email using Go's multipart parser and populates the headers, body and processes attachments
+func ParseWithAttachments(request *http.Request) (*ParsedEmail, error) {
+	result := newParsedEmail(request)
+	result.withAttachments = true
+
 	err := result.parse()
 	return &result, err
 }
@@ -32,13 +86,75 @@ func (email *ParsedEmail) parse() error {
 	if err != nil {
 		return err
 	}
-	emails := email.rawRequest.MultipartForm.Value["email"]
-	headers := email.rawRequest.MultipartForm.Value["headers"]
-	if len(headers) > 0 {
-		email.parseHeaders(headers[0])
+
+	email.rawValues = email.rawRequest.MultipartForm.Value
+
+	// unmarshal the envelope
+	if len(email.rawValues["envelope"]) > 0 {
+		if err := json.Unmarshal([]byte(email.rawValues["envelope"][0]), &email.Envelope); err != nil {
+			return err
+		}
+	}
+
+	// parse included headers
+	if len(email.rawValues["headers"]) > 0 {
+		email.parseHeaders(email.rawValues["headers"][0])
+	}
+
+	// apply the rest of the SendGrid fields to the headers map
+	for k, v := range email.rawValues {
+		if k == "text" || k == "email" || k == "headers" || k == "envelope" {
+			continue
+		}
+
+		if len(v) > 0 {
+			email.ParsedValues[k] = v[0]
+		}
 	}
-	if len(emails) > 0 {
-		email.parseRawEmail(emails[0])
+
+	// apply the plain text body
+	if len(email.rawValues["text"]) > 0 {
+		email.TextBody = email.rawValues["text"][0]
+	}
+
+	// only included if the raw box is checked
+	if len(email.rawValues["email"]) > 0 {
+		email.parseRawEmail(email.rawValues["email"][0])
+	}
+
+	// if the client chose not to parse attachments, return as is
+	if !email.withAttachments {
+		return nil
+	}
+
+	return email.parseAttachments(email.rawValues)
+}
+
+func (email *ParsedEmail) parseAttachments(values map[string][]string) error {
+	if len(values["attachment-info"]) != 1 {
+		return nil
+	}
+	// unmarshal the sendgrid parsed aspects of the email attachment into the attachment struct
+	if err := json.Unmarshal([]byte(values["attachment-info"][0]), &email.ParsedAttachments); err != nil {
+		return err
+	}
+
+	// range through the multipart files
+	for key, val := range email.rawRequest.MultipartForm.File {
+		// open the attachment file for processing
+		file, err := val[0].Open()
+		if err != nil {
+			return err
+		}
+
+		// add the actual file and the size to the parsed files
+		email.ParsedAttachments[key].File = file
+		email.ParsedAttachments[key].Size = val[0].Size
+
+		// if the file does not have a name. give it Untitled
+		if email.ParsedAttachments[key].Filename == "" {
+			email.ParsedAttachments[key].Filename = "Untitled"
+		}
 	}
 
 	return nil
@@ -87,6 +203,7 @@ func (email *ParsedEmail) parseRawEmail(rawEmail string) error {
 			if err != nil {
 				return err
 			}
+
 			email.Body[header] = string(b)
 		}
 	}
@@ -108,6 +225,32 @@ func (email *ParsedEmail) parseHeaders(headers string) {
 	splitHeaders := strings.Split(strings.TrimSpace(headers), "\n")
 	for _, header := range splitHeaders {
 		splitHeader := strings.SplitN(header, ": ", 2)
+		// keeps outlook emails from causing a panic
+		if len(splitHeader) != 2 {
+			continue
+		}
+
 		email.Headers[splitHeader[0]] = splitHeader[1]
 	}
 }
+
+// Validate validates the DKIM and SPF scores to ensure that the email client and address was not spoofed
+func (email *ParsedEmail) Validate() error {
+	if len(email.rawValues["dkim"]) == 0 || len(email.rawValues["SPF"]) == 0 {
+		return fmt.Errorf("missing DKIM and SPF score")
+	}
+
+	for _, val := range email.rawValues["dkim"] {
+		if !strings.Contains(val, "pass") {
+			return fmt.Errorf("DKIM validation failed")
+		}
+	}
+
+	for _, val := range email.rawValues["SPF"] {
+		if !strings.Contains(val, "pass") {
+			return fmt.Errorf("SPF validation failed")
+		}
+	}
+
+	return nil
+}

helpers/inbound/inbound_test.go

@@ -60,12 +60,13 @@ func TestParse(t *testing.T) {
 			email, err := Parse(req)
 			if test.expectedError != nil {
 				assert.Error(subTest, err, "expected an error to occur")
-			} else {
-				assert.NoError(subTest, err, "did NOT expect an error to occur")
-
-				from := "Example User <test@example.com>"
-				assert.Equalf(subTest, email.Headers["From"], from, "Expected From: %s, Got: %s", from, email.Headers["From"])
+				return
 			}
+
+			assert.NoError(subTest, err, "did NOT expect an error to occur")
+
+			from := "Example User <test@example.com>"
+			assert.Equalf(subTest, email.Headers["From"], from, "Expected From: %s, Got: %s", from, email.Headers["From"])
 		})
 	}
 }
@@ -129,3 +130,51 @@ Content-Transfer-Encoding: quoted-printable
 	// Content-Type multipart/mixed; boundary=TwiLIo
 	// Hello Twilio SendGrid!
 }
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		name          string
+		values        map[string][]string
+		expectedError error
+	}{
+		{
+			name:          "MissingHeaders",
+			values:        map[string][]string{},
+			expectedError: fmt.Errorf("missing DKIM and SPF score"),
+		},
+		{
+			name:          "FailedDkim",
+			values:        map[string][]string{"dkim": {"pass", "fail", "pass"}, "SPF": {"pass"}},
+			expectedError: fmt.Errorf("DKIM validation failed"),
+		},
+		{
+			name:          "FailedSpf",
+			values:        map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "fail", "pass"}},
+			expectedError: fmt.Errorf("SPF validation failed"),
+		},
+		{
+			name:          "FailedSpfandDkim",
+			values:        map[string][]string{"dkim": {"pass", "pass", "fail"}, "SPF": {"pass", "fail", "pass"}},
+			expectedError: fmt.Errorf("DKIM validation failed"),
+		},
+		{
+			name:   "success",
+			values: map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "pass", "pass"}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(subTest *testing.T) {
+			//Load POST body
+			email := ParsedEmail{rawValues: test.values}
+			err := email.Validate()
+
+			if test.expectedError != nil {
+				assert.EqualError(subTest, test.expectedError, err.Error())
+				return
+			}
+
+			assert.NoError(subTest, err, "did NOT expect an error to occur")
+		})
+	}
+}