Merge pull request #129 from Foundato/develop

Update opa dependencie and implement testing

.gitignore

@@ -11,6 +11,11 @@
 # Test binary, build with `go test -c`
 *.test
 
+# Test autogenerated files and results
+test/load/scripts
+test/load/results
+test/e2e/results
+
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 

.golangci.yml

@@ -54,12 +54,12 @@ linters:
     - errcheck
     - gochecknoinits
     - goconst
-    # - gocritic
+    - gocritic
     - gochecknoglobals
     - gocyclo
     - gofmt
     - goimports
-    # - golint
+    - golint
     - gosec
     - gosimple
     - govet

Makefile

@@ -2,16 +2,16 @@ PROJECT_NAME := "kelon"
 PKG := "github.com/Foundato/$(PROJECT_NAME)"
 PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/)
 GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go)
- 
-.PHONY: all dep lint vet test test-coverage build clean
+
+.PHONY: all dep lint vet test test-coverage build clean e2e-test load-test load-test-update-postman
 
 all: build
 
 dep: ## Get the dependencies
 	@go mod download
 
 lint: ## Lint Golang files
-	@golint -set_exit_status ${PKG_LIST}
+	@golangci-lint -c .golangci.yml run
 
 vet: ## Run go vet
 	@go vet ${PKG_LIST}
@@ -30,4 +30,45 @@ clean: ## Remove previous build
 	@rm -f $(PROJECT_NAME)/build
 
 help: ## Display this help screen
-	@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+e2e-test:
+	docker-compose up --build -d
+
+	while [[ "$$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8181/health)" != "200" ]]; do sleep 2; done
+
+	docker run -v $(PWD)/test/e2e:/etc/newman -t \
+		--network="kelon_compose_network" --rm\
+		postman/newman run Kelon_E2E.postman_collection.json \
+		--environment='kelon.postman_environment.json' \
+		--reporters cli,junit \
+		-n 5 \
+		--reporter-junit-export results/e2e-junit-results.xml || (docker-compose down --volumes; exit 1;)
+
+	docker-compose down --volumes
+	exit 0
+
+load-test:
+	docker-compose up --build -d
+
+	if [[ $$(ls ./test/load/scripts | wc -l ) -ne 4 ]]; then make load-test-update-postman; fi
+
+	while [[ "$$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8181/health)" != "200" ]]; do sleep 2; done
+
+	docker run -it -v $(PWD)/test/load:/output/ --rm --network="kelon_compose_network" loadimpact/k6 run /output/mongo_k6_load_tests.js || (docker-compose down --volumes; exit 1;)
+	docker run -it -v $(PWD)/test/load:/output/ --rm --network="kelon_compose_network" loadimpact/k6 run /output/mysql_k6_load_tests.js || (docker-compose down --volumes; exit 1;)
+	docker run -it -v $(PWD)/test/load:/output/ --rm --network="kelon_compose_network" loadimpact/k6 run /output/postgre_k6_load_tests.js || (docker-compose down --volumes; exit 1;)
+
+	docker-compose down --volumes
+
+	exit 0
+
+# run once before running load test
+load-test-update-postman:
+	docker run -it -v $(PWD)/test/load:/output/ --rm loadimpact/postman-to-k6 /output/collections/mongo_kelon_load.postman_collection.json -o /output/scripts/mongo_default_function_autogenerated.js
+	docker run -it -v $(PWD)/test/load:/output/ --rm loadimpact/postman-to-k6 /output/collections/mysql_kelon_load.postman_collection.json -o /output/scripts/mysql_default_function_autogenerated.js
+	docker run -it -v $(PWD)/test/load:/output/ --rm loadimpact/postman-to-k6 /output/collections/postgre_kelon_load.postman_collection.json -o /output/scripts/postgre_default_function_autogenerated.js
+
+
+integration-test:
+	@go test ./test/integration

cmd/kelon/kelon.go

@@ -10,7 +10,6 @@ import (
 	"github.com/Foundato/kelon/common"
 	"github.com/Foundato/kelon/configs"
 	apiInt "github.com/Foundato/kelon/internal/pkg/api"
-	"github.com/Foundato/kelon/internal/pkg/api/envoy"
 	"github.com/Foundato/kelon/internal/pkg/data"
 	opaInt "github.com/Foundato/kelon/internal/pkg/opa"
 	requestInt "github.com/Foundato/kelon/internal/pkg/request"
@@ -29,7 +28,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-//nolint:gochecknoglobals
+//nolint:gochecknoglobals,gocritic
 var (
 	app = kingpin.New("kelon", "Kelon policy enforcer.")
 
@@ -43,33 +42,22 @@ var (
 	opaPath           = app.Flag("opa-conf", "Path to the OPA configuration yaml.").Short('o').Default("./opa.yml").Envar("OPA_CONF").ExistingFile()
 	regoDir           = app.Flag("rego-dir", "Dir containing .rego files which will be loaded into OPA.").Short('r').Envar("REGO_DIR").ExistingDir()
 
-	// Additional configs
+	// Additional config
 	pathPrefix            = app.Flag("path-prefix", "Prefix which is used to proxy OPA's Data-API.").Default("/v1").Envar("PATH_PREFIX").String()
 	port                  = app.Flag("port", "Port on which the proxy endpoint is served.").Short('p').Default("8181").Envar("PORT").Uint32()
 	preprocessRegos       = app.Flag("preprocess-policies", "Preprocess incoming policies for internal use-case (EXPERIMENTAL FEATURE! DO NOT USE!).").Default("false").Envar("PREPROCESS_POLICIES").Bool()
 	respondWithStatusCode = app.Flag("respond-with-status-code", "Communicate Decision via status code 200 (ALLOW) or 403 (DENY).").Default("false").Envar("RESPOND_WITH_STATUS_CODE").Bool()
 
 	// Logging
-	logLevel  = app.Flag("log-level", "Log-Level for Kelon. Must be one of [DEBUG, INFO, WARN, ERROR]").Default("INFO").Envar("LOG_LEVEL").Enum("DEBUG", "INFO", "WARN", "ERROR", "debug", "info", "warn", "error")
-	logFormat = app.Flag("log-format", "Log-Format for Kelon. Must be one of [TEXT, JSON]").Default("TEXT").Envar("LOG_FORMAT").Enum("TEXT", "JSON")
-
-	// Configs for envoy external auth
-	envoyPort       = app.Flag("envoy-port", "Also start Envoy GRPC-Proxy on specified port so integrate kelon with Istio.").Envar("ENVOY_PORT").Uint32()
-	envoyDryRun     = app.Flag("envoy-dry-run", "Enable/Disable the dry run feature of the envoy-proxy.").Default("false").Envar("ENVOY_DRY_RUN").Bool()
-	envoyReflection = app.Flag("envoy-reflection", "Enable/Disable the reflection feature of the envoy-proxy.").Default("true").Envar("ENVOY_REFLECTION").Bool()
+	logLevel               = app.Flag("log-level", "Log-Level for Kelon. Must be one of [DEBUG, INFO, WARN, ERROR]").Default("INFO").Envar("LOG_LEVEL").Enum("DEBUG", "INFO", "WARN", "ERROR", "debug", "info", "warn", "error")
+	logFormat              = app.Flag("log-format", "Log-Format for Kelon. Must be one of [TEXT, JSON]").Default("TEXT").Envar("LOG_FORMAT").Enum("TEXT", "JSON")
+	accessDecisionLogLevel = app.Flag("access-decision-log-level", "Access decision Log-Level for Kelon. Must be one of [ALL, ALLOW, DENY, NONE]").Default("ALL").Envar("ACCESS_DECISION_LOG_LEVEL").Enum("ALL", "ALLOW", "DENY", "NONE", "all", "allow", "deny", "none")
 
 	// Configs for telemetry
-	telemetryService                = app.Flag("telemetry-service", "Service that is used for telemetry [Prometheus, ApplicationInsights]").Envar("TELEMETRY_SERVICE").Enum("Prometheus", "prometheus", "ApplicationInsights", "applicationinsights")
-	instrumentationKey              = app.Flag("instrumentation-key", "The ApplicationInsights-InstrumentationKey that is used to connect to the API.").Envar("INSTRUMENTATION_KEY").String()
-	appInsightsServiceName          = app.Flag("application-insights-service-name", "The name which will be displayed for kelon inside application insights.").Default("Kelon").Envar("APPLICATION_INSIGHTS_SERVICE_NAME").String()
-	appInsightsMaxBatchSize         = app.Flag("application-insights-max-batch-size", "Configure how many items can be sent in one call to the data collector.").Default("8192").Envar("APPLICATION_INSIGHTS_MAX_BATCH_SIZE").Int()
-	appInsightsMaxBatchInterval     = app.Flag("application-insights-max-batch-interval-seconds", "Configure the maximum delay before sending queued telemetry.").Default("2").Envar("APPLICATION_INSIGHTS_MAX_BATCH_INTERVAL_SECONDS").Int()
-	appInsightsLogLevels            = app.Flag("application-insights-log-levels", "Configure log levels which are sent. Allowed values are [fatal, panic, error, warn, info, debug, trace]").Default("fatal,panic,error,warn").Envar("APPLICATION_INSIGHTS_LOG_LEVELS").String()
-	appInsightsStatsIntervalSeconds = app.Flag("application-insights-stats-interval-seconds", "Interval in seconds in which system stats are measured and sent.").Default("5").Envar("APPLICATION_INSIGHTS_STATS_INTERVAL_SECONDS").Int()
+	telemetryService = app.Flag("telemetry-service", "Service that is used for telemetry [Prometheus]").Envar("TELEMETRY_SERVICE").Enum("Prometheus", "prometheus")
 
 	// Global shared variables
 	proxy             api.ClientProxy       = nil
-	envoyProxy        api.ClientProxy       = nil
 	configWatcher     watcher.ConfigWatcher = nil
 	telemetryProvider telemetry.Provider    = nil
 )
@@ -138,7 +126,7 @@ func onConfigLoaded(change watcher.ChangeType, loadedConf *configs.ExternalConfi
 			translator = translateInt.NewAstTranslator()
 		)
 
-		// Build configs
+		// Build config
 		config.API = loadedConf.API
 		config.Data = loadedConf.Data
 		config.TelemetryProvider = makeTelemetryProvider()
@@ -151,29 +139,14 @@ func onConfigLoaded(change watcher.ChangeType, loadedConf *configs.ExternalConfi
 
 		// Start rest proxy
 		startNewRestProxy(config, &serverConf)
-
-		// Start envoyProxy proxy in addition to rest proxy as soon as a port was specified!
-		if envoyPort != nil && *envoyPort != 0 {
-			startNewEnvoyProxy(config, &serverConf)
-		}
 	}
 }
 
 func makeTelemetryProvider() telemetry.Provider {
 	var provider telemetry.Provider
 	if telemetryService != nil {
-		switch strings.ToLower(*telemetryService) {
-		case constants.PrometheusTelemetry:
+		if strings.EqualFold(*telemetryService, constants.PrometheusTelemetry) {
 			provider = &telemetry.Prometheus{}
-		case constants.ApplicationInsightsTelemetry:
-			provider = &telemetry.ApplicationInsights{
-				AppInsightsInstrumentationKey: *instrumentationKey,
-				ServiceName:                   *appInsightsServiceName,
-				MaxBatchSize:                  *appInsightsMaxBatchSize,
-				MaxBatchIntervalSeconds:       *appInsightsMaxBatchInterval,
-				LogLevels:                     *appInsightsLogLevels,
-				StatsIntervalSeconds:          *appInsightsStatsIntervalSeconds,
-			}
 		}
 
 		if provider != nil {
@@ -209,26 +182,6 @@ func startNewRestProxy(appConfig *configs.AppConfig, serverConf *api.ClientProxy
 	}
 }
 
-func startNewEnvoyProxy(appConfig *configs.AppConfig, serverConf *api.ClientProxyConfig) {
-	if *envoyPort == *port {
-		logging.LogForComponent("main").Panic("Cannot start envoyProxy proxy and rest proxy on same port!")
-	}
-
-	// Create Rest proxy and start
-	envoyProxy = envoy.NewEnvoyProxy(envoy.EnvoyConfig{
-		Port:             *envoyPort,
-		DryRun:           *envoyDryRun,
-		EnableReflection: *envoyReflection,
-	})
-	if err := envoyProxy.Configure(appConfig, serverConf); err != nil {
-		logging.LogForComponent("main").Fatalln(err.Error())
-	}
-	// Start proxy
-	if err := envoyProxy.Start(); err != nil {
-		logging.LogForComponent("main").Fatalln(err.Error())
-	}
-}
-
 func makeServerConfig(compiler opa.PolicyCompiler, parser request.PathProcessor, mapper request.PathMapper, translator translate.AstTranslator, loadedConf *configs.ExternalConfig) api.ClientProxyConfig {
 	// Build server config
 	serverConf := api.ClientProxyConfig{
@@ -247,6 +200,7 @@ func makeServerConfig(compiler opa.PolicyCompiler, parser request.PathProcessor,
 			AstTranslatorConfig: translate.AstTranslatorConfig{
 				Datastores: data.MakeDatastores(loadedConf.Data),
 			},
+			AccessDecisionLogLevel: strings.ToUpper(*accessDecisionLogLevel),
 		},
 	}
 	return serverConf
@@ -266,13 +220,6 @@ func stopOnSIGTERM() {
 		telemetryProvider.Shutdown()
 	}
 
-	// Stop envoyProxy proxy if started
-	if envoyProxy != nil {
-		if err := envoyProxy.Stop(time.Second * 10); err != nil {
-			logging.LogForComponent("main").Warnln(err.Error())
-		}
-	}
-
 	// Stop rest proxy if started
 	if proxy != nil {
 		if err := proxy.Stop(time.Second * 10); err != nil {

common/version.go

@@ -2,5 +2,5 @@
 package common
 
 // Version of the running kelon instance (probably set by CI-pipeline)
-//nolint:gochecknoglobals
+//nolint:gocritic,gochecknoglobals
 var Version = "0.3.0"

configs/config.go

@@ -1,4 +1,4 @@
-// Central package for app-global configs.
+// Central package for app-global config.
 package configs
 
 import (
@@ -16,7 +16,7 @@ type AppConfig struct {
 	TelemetryProvider telemetry.Provider
 }
 
-// External configs.
+// External config.
 type ExternalConfig struct {
 	Data *DatastoreConfig
 	API  *APIConfig
@@ -30,7 +30,7 @@ type ConfigLoader interface {
 	Load() (*ExternalConfig, error)
 }
 
-// ByteConfigLoader implements configs.ConfigLoader by loading configs from
+// ByteConfigLoader implements configs.ConfigLoader by loading config from
 // two provided bytes slices.
 type ByteConfigLoader struct {
 	DatastoreConfigBytes []byte
@@ -40,10 +40,10 @@ type ByteConfigLoader struct {
 // Implementing Load from configs.ConfigLoader by using the properties of the ByteConfigLoader.
 func (l ByteConfigLoader) Load() (*ExternalConfig, error) {
 	if l.DatastoreConfigBytes == nil {
-		return nil, errors.New("DatastoreConfigBytes must not be nil! ")
+		return nil, errors.Errorf("DatastoreConfigBytes must not be nil!")
 	}
 	if l.APIConfigBytes == nil {
-		return nil, errors.New("APIConfigBytes must not be nil! ")
+		return nil, errors.Errorf("APIConfigBytes must not be nil! ")
 	}
 
 	result := new(ExternalConfig)
@@ -53,24 +53,24 @@ func (l ByteConfigLoader) Load() (*ExternalConfig, error) {
 	// Expand datastore config with environment variables
 	l.DatastoreConfigBytes = []byte(os.ExpandEnv(string(l.DatastoreConfigBytes)))
 	if err := yaml.Unmarshal(l.DatastoreConfigBytes, result.Data); err != nil {
-		return nil, errors.New("Unable to parse datastore config: " + err.Error())
+		return nil, errors.Errorf("Unable to parse datastore config: " + err.Error())
 	}
 
 	// Load API config
 	result.API = new(APIConfig)
 	if err := yaml.Unmarshal(l.APIConfigBytes, result.API); err != nil {
-		return nil, errors.New("Unable to parse api config: " + err.Error())
+		return nil, errors.Errorf("Unable to parse api config: " + err.Error())
 	}
 
-	// Validate configs
+	// Validate config
 	if err := result.Data.validate(); err != nil {
 		return nil, errors.Wrap(err, "Loaded invalid datastore config")
 	}
 
 	return result, nil
 }
 
-// FileConfigLoader implements configs.ConfigLoader by loading configs from
+// FileConfigLoader implements configs.ConfigLoader by loading config from
 // two files located at given paths.
 type FileConfigLoader struct {
 	DatastoreConfigPath string
@@ -80,10 +80,10 @@ type FileConfigLoader struct {
 // Implementing Load from configs.ConfigLoader by using the properties of the FileConfigLoader.
 func (l FileConfigLoader) Load() (*ExternalConfig, error) {
 	if l.DatastoreConfigPath == "" {
-		return nil, errors.New("DatastoreConfigPath must not be empty! ")
+		return nil, errors.Errorf("DatastoreConfigPath must not be empty!")
 	}
 	if l.APIConfigPath == "" {
-		return nil, errors.New("APIConfigPath must not be empty! ")
+		return nil, errors.Errorf("APIConfigPath must not be empty! ")
 	}
 
 	// Load dsConfigBytes from file

configs/config_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-//nolint:gochecknoglobals
+//nolint:gochecknoglobals,gocritic
 var wantDatatoreConfig = configs.DatastoreConfig{
 	Datastores: map[string]*configs.Datastore{
 		"mysql": {
@@ -51,7 +51,7 @@ var wantDatatoreConfig = configs.DatastoreConfig{
 	},
 }
 
-//nolint:gochecknoglobals
+//nolint:gochecknoglobals,gocritic
 var wantAPIConfig = &configs.APIConfig{
 	Mappings: []*configs.DatastoreAPIMapping{
 		{

configs/datastore.go

@@ -73,12 +73,14 @@ func (schema EntitySchema) GenerateEntityPaths() map[string]map[string][]string
 	return result
 }
 
-func crawlEntityPaths(start *Entity, curr *Entity, pathHistory []string, path map[string]map[string][]string) {
+func crawlEntityPaths(start, curr *Entity, pathHistory []string, path map[string]map[string][]string) {
 	if start == nil {
 		logging.LogForComponent("datastore").Panic("Cannot crawl start which is nil!")
+		return
 	}
 	if curr == nil {
 		logging.LogForComponent("datastore").Panic("Curr mustn't be nil!")
+		return
 	}
 
 	if path[start.getMappedName()] == nil {