Merge pull request #62 from svanharmelen/dev

Lot's of changes, please see the CHANGELOG for details

CHANGELOG.md

@@ -1,11 +1,22 @@
 Chef-Guard CHANGELOG
 ====================
 
+0.5.0
+-----
+- Fixed an issue where file handles where not always released correctly
+- Added a check to verify if cfg.Community.Forks is actually configued before using it
+- Refactored the fix for issue #53 as the current fix wasn't effecient and caused 'too many open files' issues
+- Added a check to make sure all custom Github endpoints end with a single forward slash
+- Added a configuration option to specify a custom sender address used when sending the diffs
+- Added an additional (non-Chef) endpoint to download clients from
+- Added an additional (non-Chef) endpoint to get the server time from
+- Fixed a bug when running the Rubocop tests
+
 0.4.5
 -----
 - Added the '.json' extention to cookbook auditing files saved in Github to have uniform names
 - Fixed issue #53 by making sure the config is checked and used to determine if we want to verify SSL
-- Fixed issue #54 by adding a check if a value is actually configued before using it
+- Fixed issue #54 by adding a check to verify if a value is actually configued before using it
 - Added code to check if the config file contains values for all required fields
 
 0.4.4

VERSION

@@ -1 +1 @@
-0.4.5
+0.5.0

checks.go

@@ -84,13 +84,15 @@ func getFoodcriticArgs(org, cookbookPath string) []string {
 
 func runRubocop(cookbookPath string) (int, error) {
 	cmd := exec.Command(cfg.Tests.Rubocop, cookbookPath)
+	cmd.Env = []string{"HOME=" + cfg.Default.Tempdir}
 	output, err := cmd.CombinedOutput()
 	if err != nil {
-		return http.StatusBadGateway, fmt.Errorf("Failed to execute rubocop tests: %s - %s", output, err)
-	}
-	if strings.TrimSpace(string(output)) != "" {
-		errText := strings.TrimSpace(strings.Replace(string(output), fmt.Sprintf("%s/", cookbookPath), "", -1))
-		return http.StatusPreconditionFailed, fmt.Errorf("\n=== Rubocop errors found ===\n%s\n============================\n", errText)
+		if strings.Contains(string(output), "offenses detected") {
+			errText := strings.TrimSpace(strings.Replace(string(output), fmt.Sprintf("%s/", cookbookPath), "", -1))
+			return http.StatusPreconditionFailed, fmt.Errorf("\n=== Rubocop errors found ===\n%s\n============================\n", errText)
+		} else {
+			return http.StatusBadGateway, fmt.Errorf("Failed to execute rubocop tests: %s - %s", output, err)
+		}
 	}
 	return 0, nil
 }

chef-guard.go

@@ -115,28 +115,21 @@ func main() {
 		rtr.Path("/organizations/{org}/{type:environments|nodes|roles}").HandlerFunc(processChange(p)).Methods("POST")
 		rtr.Path("/organizations/{org}/{type:environments|nodes|roles}/{name}").HandlerFunc(processChange(p)).Methods("PUT", "DELETE")
 		rtr.Path("/organizations/{org}/{type:cookbooks}/{name}/{version}").HandlerFunc(processCookbook(p)).Methods("PUT", "DELETE")
-
-		rtr.Path("/organizations/chef-guard/time").HandlerFunc(timeHandler).Methods("GET")
-		if cfg.ChefClients.Path != "" {
-			rtr.Path("/organizations/chef-guard/metadata").HandlerFunc(metadataHandler).Methods("GET")
-			rtr.Path("/organizations/chef-guard/download").HandlerFunc(downloadHandler).Methods("GET")
-			rtr.PathPrefix("/organizations/chef-guard/clients").Handler(http.StripPrefix("/organizations/chef-guard/clients/", http.FileServer(http.Dir(cfg.ChefClients.Path))))
-		}
-
 	} else {
 		rtr.Path("/{type:data}/{bag}").HandlerFunc(processChange(p)).Methods("POST", "DELETE")
 		rtr.Path("/{type:data}/{bag}/{name}").HandlerFunc(processChange(p)).Methods("PUT", "DELETE")
 		rtr.Path("/{type:environments|nodes|roles}").HandlerFunc(processChange(p)).Methods("POST")
 		rtr.Path("/{type:environments|nodes|roles}/{name}").HandlerFunc(processChange(p)).Methods("PUT", "DELETE")
 		rtr.Path("/{type:cookbooks}/{name}/{version}").HandlerFunc(processCookbook(p)).Methods("PUT", "DELETE")
+	}
 
-		rtr.Path("/chef-guard/time").HandlerFunc(timeHandler).Methods("GET")
-		if cfg.ChefClients.Path != "" {
-			rtr.Path("/chef-guard/metadata").HandlerFunc(metadataHandler).Methods("GET")
-			rtr.Path("/chef-guard/download").HandlerFunc(downloadHandler).Methods("GET")
-			rtr.PathPrefix("/chef-guard/clients").Handler(http.StripPrefix("/chef-guard/clients/", http.FileServer(http.Dir(cfg.ChefClients.Path))))
-		}
+	// Adding some non-Chef endpoints here
+	rtr.Path("/chef-guard/time").HandlerFunc(timeHandler).Methods("GET")
+	if cfg.ChefClients.Path != "" {
+		rtr.Path("/chef-guard/{type:metadata|download}").HandlerFunc(processDownload).Methods("GET")
+		rtr.PathPrefix("/chef-guard/clients").Handler(http.StripPrefix("/chef-guard/clients/", http.FileServer(http.Dir(cfg.ChefClients.Path))))
 	}
+
 	rtr.NotFoundHandler = p
 	http.Handle("/", rtr)
 

chefclients.go → clients.go

@@ -21,15 +21,16 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
+
+	"github.com/gorilla/mux"
 )
 
-// Add type and functions for the Sort interface
+// Add files type and functions for the Sort interface
 type files []string
 
 func (f files) Len() int {
@@ -64,91 +65,56 @@ func (f files) Swap(i, j int) {
 	f[i], f[j] = f[j], f[i]
 }
 
-func getURLParams(r *http.Request) string {
-	return filepath.Join(r.FormValue("p"), r.FormValue("pv"), r.FormValue("m"))
-}
+func processDownload(w http.ResponseWriter, r *http.Request) {
+	path := getFilePath(r)
+	dir := filepath.Join(cfg.ChefClients.Path, path)
+
+	targetfile, err := getTargetFile(dir, r.FormValue("v"))
+	if err != nil {
+		errorHandler(w, err.Error(), http.StatusBadGateway)
+	}
 
-func getTargetDir(r *http.Request) string {
-	params := getURLParams(r)
+	if targetfile != "" {
+		targetpath := path + targetfile[len(dir):]
+		targeturl := getChefBaseURL() + "/chef-guard/clients/" + targetpath
 
-	if cfg.ChefClients.Path != "" {
-		params = filepath.Join(cfg.ChefClients.Path, params)
+		// For download calls, redirect to the actuall file
+		if mux.Vars(r)["type"] == "download" {
+			http.Redirect(w, r, targeturl, http.StatusFound)
+		}
+		// For metadata calls, return the requested meta data
+		if mux.Vars(r)["type"] == "metadata" {
+			data, err := ioutil.ReadFile(targetfile)
+			if err != nil {
+				errorHandler(w, "Failed to read client file: %s"+err.Error(), http.StatusBadGateway)
+			}
+
+			targetmd5 := md5.Sum(data)
+			targetsha := sha256.Sum256(data)
+			data = nil
+
+			fmt.Fprintf(w, "url %s md5 %x sha256 %x", targeturl, targetmd5, targetsha)
+		}
 	}
-	return params
 }
 
-func getTargetFile(r *http.Request) string {
-	dir := getTargetDir(r)
+func getFilePath(r *http.Request) string {
+	return filepath.Join(r.FormValue("p"), r.FormValue("pv"), r.FormValue("m"))
+}
 
-	version := r.FormValue("v")
+func getTargetFile(dir, version string) (string, error) {
 	if version == "latest" {
 		version = "."
 	}
 
-	filelist, _ := filepath.Glob(dir + "/*" + version + "*")
+	filelist, err := filepath.Glob(dir + "/*" + version + "*")
+	if err != nil {
+		return "", fmt.Errorf("Failed to read clients from disk: %s", err)
+	}
 
 	if filelist != nil {
 		sort.Sort(files(filelist))
-		return filelist[0]
-	} else {
-		return ""
-	}
-}
-
-func getTargetURL(target string) string {
-	host := cfg.Chef.Server
-	switch cfg.Chef.Port {
-	case "443":
-		host = "https://" + host
-	case "80":
-		host = "http://" + host
-	case "":
-		host = "http://" + host
-	default:
-		host = "http://" + host + ":" + cfg.Chef.Port
-	}
-
-	if cfg.Chef.EnterpriseChef {
-		// BASEURL needs to be fixed
-		return host + "/organizations/chef-guard/clients/" + target
-	} else {
-		return host + "/chef-guard/clients/" + target
-	}
-}
-
-func downloadHandler(w http.ResponseWriter, r *http.Request) {
-	path := getURLParams(r)
-	dir := getTargetDir(r)
-
-	targetfile := getTargetFile(r)
-
-	if targetfile != "" {
-		targetpath := path + targetfile[len(dir):]
-
-		targeturl := getTargetURL(targetpath)
-		http.Redirect(w, r, targeturl, http.StatusFound)
-	}
-}
-
-func metadataHandler(w http.ResponseWriter, r *http.Request) {
-	path := getURLParams(r)
-	dir := getTargetDir(r)
-
-	targetfile := getTargetFile(r)
-
-	if targetfile != "" {
-		targetpath := path + targetfile[len(dir):]
-		targeturl := getTargetURL(targetpath)
-
-		data, err := ioutil.ReadFile(targetfile)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		targetmd5 := md5.Sum(data)
-		targetsha := sha256.Sum256(data)
-		data = nil
-
-		fmt.Fprintf(w, "url %s md5 %x sha256 %x", targeturl, targetmd5, targetsha)
+		return filelist[0], nil
 	}
+	return "", nil
 }

config.go

@@ -37,6 +37,7 @@ type Config struct {
 		MailDomain      string
 		MailServer      string
 		MailPort        int
+		MailSendBy      string
 		MailRecipient   string
 		ValidateChanges string
 		SaveChefMetrics bool
@@ -55,6 +56,7 @@ type Config struct {
 		MailDomain      *string
 		MailServer      *string
 		MailPort        *int
+		MailSendBy      *string
 		MailRecipient   *string
 		ValidateChanges *string
 		SaveChefMetrics *bool

cookbook.go

@@ -107,8 +107,13 @@ func (cg *ChefGuard) processCookbookFiles() error {
 	gw := gzip.NewWriter(buf)
 	tw := tar.NewWriter(gw)
 
+	t := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: cfg.Chef.SSLNoVerify},
+	}
+	c := &http.Client{Transport: t}
+
 	for _, f := range cg.getAllCookbookFiles() {
-		content, err := downloadCookbookFile(*cg.OrganizationID, f.Checksum)
+		content, err := downloadCookbookFile(c, *cg.OrganizationID, f.Checksum)
 		if err != nil {
 			return fmt.Errorf("Failed to dowload %s from the %s cookbook: %s", f.Path, cg.Cookbook.Name, err)
 		}
@@ -241,23 +246,22 @@ func (cg *ChefGuard) getCookbookChangeDetails(r *http.Request) []byte {
 	return []byte(details)
 }
 
-func downloadCookbookFile(orgID, checksum string) ([]byte, error) {
+func downloadCookbookFile(c *http.Client, orgID, checksum string) ([]byte, error) {
 	u, err := generateSignedURL(orgID, checksum)
 	if err != nil {
 		return nil, err
 	}
-	t := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: cfg.Chef.SSLNoVerify},
-	}
-	c := &http.Client{Transport: t}
 	resp, err := c.Get(u.String())
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+
 	if err := checkHTTPResponse(resp, []int{http.StatusOK}); err != nil {
 		return nil, err
 	}
-	return dumpBody(resp)
+
+	return ioutil.ReadAll(resp.Body)
 }
 
 func generateSignedURL(orgID, checksum string) (*url.URL, error) {
@@ -268,16 +272,7 @@ func generateSignedURL(orgID, checksum string) (*url.URL, error) {
 	h.Write([]byte(stringToSign))
 	signature := url.QueryEscape(base64.StdEncoding.EncodeToString(h.Sum(nil)))
 
-	var baseURL string
-	switch cfg.Chef.Port {
-	case "443":
-		baseURL = fmt.Sprintf("https://%s", cfg.Chef.Server)
-	case "80":
-		baseURL = fmt.Sprintf("http://%s", cfg.Chef.Server)
-	default:
-		baseURL = fmt.Sprintf("%s:%s", cfg.Chef.Server, cfg.Chef.Port)
-	}
-	return url.Parse(fmt.Sprintf("%s/bookshelf/organization-%s/checksum-%s?AWSAccessKeyId=%s&Expires=%d&Signature=%s", baseURL, orgID, checksum, cfg.Chef.S3Key, expires, signature))
+	return url.Parse(fmt.Sprintf("%s/bookshelf/organization-%s/checksum-%s?AWSAccessKeyId=%s&Expires=%d&Signature=%s", getChefBaseURL(), orgID, checksum, cfg.Chef.S3Key, expires, signature))
 }
 
 func writeFileToDisk(filePath string, content io.Reader) error {
@@ -288,10 +283,11 @@ func writeFileToDisk(filePath string, content io.Reader) error {
 	if err != nil {
 		return err
 	}
+	defer fo.Close()
+
 	if _, err := io.Copy(fo, content); err != nil {
 		return err
 	}
-	fo.Close()
 	return nil
 }
 
@@ -345,6 +341,19 @@ func checkHTTPResponse(resp *http.Response, allowedStates []int) error {
 	return fmt.Errorf(string(body))
 }
 
+func getChefBaseURL() string {
+	var baseURL string
+	switch cfg.Chef.Port {
+	case "443":
+		baseURL = "https://" + cfg.Chef.Server
+	case "80":
+		baseURL = "http://" + cfg.Chef.Server
+	default:
+		baseURL = cfg.Chef.Server + ":" + cfg.Chef.Port
+	}
+	return baseURL
+}
+
 func dumpBody(r interface{}) (body []byte, err error) {
 	switch r.(type) {
 	case *http.Request:

chef-guard.conf.example → examples/chef-guard.conf.example

@@ -2,22 +2,23 @@
   listen          = 127.0.0.2
   logfile         = /var/log/chef-guard.log
   tempdir         = /var/tmp/chef-guard
-  mode            = silent      # Valid options are 'silent', 'permissive' and 'enforced'
+  mode            = silent        # Valid options are 'silent', 'permissive' and 'enforced'
   maildomain      = company.com
   mailserver      = smtp.company.com
   mailport        = 25
+  mailsendby      =               # Leave blank to dynamically use the mailaddress of the user making the API call (preferred)
   mailrecipient   = chef-changes@company.com
-  validatechanges = silent      # Valid options are 'silent', 'permissive' and 'enforced'
+  validatechanges = silent        # Valid options are 'silent', 'permissive' and 'enforced'
   savechefmetrics = true
   commitchanges   = false
   mailchanges     = true
   searchgithub    = true
   publishcookbook = true
-  blacklist       =      # This can be multiple regexes divided by a ','
+  blacklist       =               # This can be multiple regexes divided by a ','
   gitorganization = chef-guard
-  gitcookbookorgs = org1, org2 # When using multiple orgs (divided by a ','), the order here determines the lookup order!
-  includefcs      =      # This should be the full path to a custom .rb file containing your custom checks
-  excludefcs      =      # This can be multiple FC's divided by a ','
+  gitcookbookorgs = org1, org2    # When using multiple orgs (divided by a ','), the order here determines the lookup order!
+  includefcs      =               # This should be the full path to a custom .rb file containing your custom checks
+  excludefcs      =               # This can be multiple FC's divided by a ','
 
 [chef]
   enterprisechef  = true
@@ -63,7 +64,7 @@
   rubocop         = /opt/chef/embedded/bin/rubocop
 
 [github "chef-guard"]
-  serverurl       =       # Empty means that it will use github.com
+  serverurl       =          # Empty means that it will use github.com
   sslnoverify     = false
   token           = xxx
 
@@ -79,4 +80,4 @@
 
 [customer "demo2"]
   mode            = enforced
-  gitcookbookorgs = demo2 # If a customer org is used in conjunction with default org(s), The default orgs are searched first!
+  gitcookbookorgs = demo2    # If a customer org is used in conjunction with default org(s), the default orgs are searched first!