config.go

// Copyright © 2022 Kaleido, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/hyperledger/firefly-common/pkg/fftypes"
	"github.com/hyperledger/firefly-common/pkg/i18n"
	"github.com/hyperledger/firefly-common/pkg/log"
	"github.com/sirupsen/logrus"
	"github.com/spf13/viper"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"
)

var ffc = AddRootKey

// The following keys can be access from the root configuration.
// Plugins are responsible for defining their own keys using the Config interface
var (
	// Lang is the language to use for translation
	Lang = ffc("lang")
	// LogForceColor forces color to be enabled, even if we do not detect a TTY
	LogForceColor = ffc("log.forceColor")
	// LogLevel is the logging level
	LogLevel = ffc("log.level")
	// LogNoColor forces color to be disabled, even if we detect a TTY
	LogNoColor = ffc("log.noColor")
	// LogTimeFormat is a string format for timestamps
	LogTimeFormat = ffc("log.timeFormat")
	// LogUTC sets log timestamps to the UTC timezone
	LogUTC = ffc("log.utc")
	// LogFilename sets logging to file
	LogFilename = ffc("log.filename")
	// LogFilesize sets the size to roll logs at
	LogFilesize = ffc("log.filesize")
	// LogMaxBackups sets the maximum number of old files to keep
	LogMaxBackups = ffc("log.maxBackups")
	// LogMaxAge sets the maximum age at which to roll
	LogMaxAge = ffc("log.maxAge")
	// LogCompress sets whether to compress backups
	LogCompress = ffc("log.compress")
	// LogIncludeCodeInfo enables the report caller for including the calling file and line number
	LogIncludeCodeInfo = ffc("log.includeCodeInfo")
	// LogJSONEnabled enables JSON formatted logs rather than text
	LogJSONEnabled = ffc("log.json.enabled")
	// LogJSONTimestampField configures the JSON key containing the timestamp of the log
	LogJSONTimestampField = ffc("log.json.fields.timestamp")
	// LogJSONLevelField configures the JSON key containing the log level
	LogJSONLevelField = ffc("log.json.fields.level")
	// LogJSONMessageField configures the JSON key containing the log message
	LogJSONMessageField = ffc("log.json.fields.message")
	// LogJSONFuncField configures the JSON key containing the calling function
	LogJSONFuncField = ffc("log.json.fields.func")
	// LogJSONFileField configures the JSON key containing the calling file
	LogJSONFileField = ffc("log.json.fields.file")
)

type KeySet interface {
	AddKnownKey(key string, defValue ...interface{})
}

type sectionParent interface {
	AddChild(key string, defValue ...interface{})
}

// Section represents a section of the global configuration, at a nested point in the config hierarchy.
// Note that all keys are added to a GLOBAL map, so this cannot be used for per-instance customization.
type Section interface {
	KeySet
	SetDefault(key string, defValue interface{})
	SubSection(name string) Section
	SubArray(name string) ArraySection
	Set(key string, value interface{})
	Resolve(key string) string

	GetString(key string) string
	GetBool(key string) bool
	GetInt(key string) int
	GetInt64(key string) int64
	GetFloat64(key string) float64
	GetByteSize(key string) int64
	GetUint(key string) uint
	GetDuration(key string) time.Duration
	GetStringSlice(key string) []string
	GetObject(key string) fftypes.JSONObject
	GetObjectArray(key string) fftypes.JSONObjectArray
	Get(key string) interface{}
}

// ArraySection represents an array of options at a particular layer in the config.
// This allows specifying the schema of keys that exist for every entry, and the defaults,
// as well as querying how many entries exist and iterating through them.
type ArraySection interface {
	KeySet
	ArraySize() int
	ArrayEntry(i int) Section
	SubSection(name string) Section
}

// RootKey key are the known configuration keys
type RootKey string

func RootConfigReset(setServiceDefaults ...func()) {
	keysMutex.Lock() // must only call viper directly here (as we already hold the lock)
	defer keysMutex.Unlock()

	viper.Reset()

	viper.SetDefault(string(Lang), "en")
	viper.SetDefault(string(LogLevel), "info")
	viper.SetDefault(string(LogTimeFormat), "2006-01-02T15:04:05.000Z07:00")
	viper.SetDefault(string(LogUTC), false)
	viper.SetDefault(string(LogFilesize), "100m")
	viper.SetDefault(string(LogMaxAge), "24h")
	viper.SetDefault(string(LogMaxBackups), 2)
	viper.SetDefault(string(LogIncludeCodeInfo), false)
	viper.SetDefault(string(LogJSONEnabled), false)
	viper.SetDefault(string(LogJSONTimestampField), "@timestamp")
	viper.SetDefault(string(LogJSONLevelField), "level")
	viper.SetDefault(string(LogJSONMessageField), "message")
	viper.SetDefault(string(LogJSONFuncField), "func")
	viper.SetDefault(string(LogJSONFileField), "file")

	// We set the service defaults within our mutex
	for _, fn := range setServiceDefaults {
		fn()
	}

	i18n.SetLang(viper.GetString(string(Lang)))
}

// ReadConfig initializes the config
func ReadConfig(cfgSuffix, cfgFile string) error {
	keysMutex.Lock() // must only call viper directly here (as we already hold the lock)
	defer keysMutex.Unlock()

	// Set precedence order for reading config location
	viper.SetEnvPrefix("firefly")
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.AutomaticEnv()
	viper.SetConfigType("yaml")
	if cfgFile != "" {
		f, err := os.Open(cfgFile)
		if err == nil {
			defer f.Close()
			err = viper.ReadConfig(f)
		}
		return err
	}
	viper.SetConfigName(fmt.Sprintf("firefly.%s", cfgSuffix))
	viper.AddConfigPath("/etc/firefly/")
	viper.AddConfigPath("$HOME/.firefly")
	viper.AddConfigPath(".")
	return viper.ReadInConfig()
}

func MergeConfig(configRecords []*fftypes.ConfigRecord) error {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	for _, c := range configRecords {
		s := viper.New()
		s.SetConfigType("json")
		var val interface{}
		if c.Value != nil {
			if err := json.Unmarshal([]byte(*c.Value), &val); err != nil {
				return err
			}
		}
		switch v := val.(type) {
		case map[string]interface{}:
			_ = s.ReadConfig(bytes.NewBuffer([]byte(*c.Value)))
			for _, k := range s.AllKeys() {
				value := s.Get(k)
				if reflect.TypeOf(value).Kind() == reflect.Slice {
					configSlice := value.([]interface{})
					for i := range configSlice {
						viper.Set(fmt.Sprintf("%s.%s.%d", c.Key, k, i), configSlice[i])
					}
				} else {
					viper.Set(fmt.Sprintf("%s.%s", c.Key, k), value)
				}
			}
		case []interface{}:
			_ = s.ReadConfig(bytes.NewBuffer([]byte(*c.Value)))
			for i := range v {
				viper.Set(fmt.Sprintf("%s.%d", c.Key, i), v[i])
			}
		default:
			viper.Set(c.Key, v)
		}
	}
	return nil
}

var knownKeys = map[string]bool{} // All config keys go here, including those defined in sub-sections
var keysMutex sync.Mutex
var root = &configSection{}

// AddRootKey adds a root key, used to define the keys that are used within the core
func AddRootKey(k string) RootKey {
	root.AddKnownKey(k)
	return RootKey(k)
}

// GetKnownKeys gets the known keys
func GetKnownKeys() []string {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	keys := make([]string, 0, len(knownKeys))
	for k := range knownKeys {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// configSection is the main config structure passed to plugins, and used for root to wrap viper
type configSection struct {
	prefix     string
	parent     sectionParent
	arrayEntry bool
}

// configArray is a point in the config that supports an array
type configArray struct {
	base     string
	parent   sectionParent
	defaults map[string][]interface{}
}

// RootSection creates a new configuration section starting from the config root
func RootSection(prefix string) Section {
	return &configSection{
		prefix: strings.TrimSuffix(prefix, "."),
	}
}

// RootArray creates a new configuration array starting from the config root
func RootArray(prefix string) ArraySection {
	return &configArray{
		base:     strings.TrimSuffix(prefix, "."),
		defaults: make(map[string][]interface{}),
	}
}

func keyName(prefix, name string) string {
	if prefix != "" {
		return prefix + "." + name
	}
	return name
}

func (c *configSection) prefixKey(k string) string {
	// Caller responsible for holding lock when calling
	key := keyName(c.prefix, k)
	if !knownKeys[key] {
		panic(fmt.Sprintf("Undefined configuration key '%s'", key))
	}
	return key
}

func (c *configSection) SubSection(name string) Section {
	return &configSection{
		prefix: keyName(c.prefix, name),
		parent: c,
	}
}

func (c *configArray) SubSection(name string) Section {
	cp := &configSection{
		prefix: keyName(c.base+"[]", name),
		parent: c,
	}
	return cp
}

// If any parent is an array entry, retrieve fully-qualified defaults via its array parent
func getArrayEntryDefaults(p sectionParent) map[string][]interface{} {
	switch c := p.(type) {
	case *configArray:
		return getArrayEntryDefaults(c.parent)
	case *configSection:
		if !c.arrayEntry {
			return getArrayEntryDefaults(c.parent)
		}
		arrayParent := c.parent.(*configArray)
		defaults := make(map[string][]interface{})
		for k, v := range arrayParent.defaults {
			defaults[c.prefix+"."+k] = v
		}
		return defaults
	default:
		return nil
	}
}

func (c *configSection) SubArray(name string) ArraySection {
	a := &configArray{
		base:     keyName(c.prefix, name),
		parent:   c,
		defaults: make(map[string][]interface{}),
	}
	// Get defaults from any enclosing array entry, and copy over any applicable to this subtree
	// This is necessary to propagate known keys for arrays within arrays
	prefix := a.base + "[]."
	for key, val := range getArrayEntryDefaults(c) {
		if strings.HasPrefix(key, prefix) {
			a.defaults[strings.TrimPrefix(key, prefix)] = val
		}
	}
	return a
}

func (c *configArray) ArraySize() int {
	val := viper.Get(c.base)
	vt := reflect.TypeOf(val)
	if vt != nil && (vt.Kind() == reflect.Slice || vt.Kind() == reflect.Map) {
		return reflect.ValueOf(val).Len()
	}
	return 0
}

// ArrayEntry must only be called after the config has been loaded
func (c *configArray) ArrayEntry(i int) Section {
	cp := &configSection{
		prefix:     keyName(c.base, fmt.Sprintf("%d", i)),
		parent:     c,
		arrayEntry: true,
	}
	for knownKey, defValue := range c.defaults {
		cp.AddKnownKey(knownKey, defValue...)
		// Sadly Viper can't handle defaults inside the array, when
		// a value is set. So here we check/set the defaults.
		if defValue != nil && cp.Get(knownKey) == nil {
			if len(defValue) == 1 {
				cp.Set(knownKey, defValue[0])
			} else if len(defValue) > 0 {
				cp.Set(knownKey, defValue)
			}
		}
	}
	return cp
}

func (c *configArray) AddKnownKey(k string, defValue ...interface{}) {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	// Put a simulated key in the known keys array, to pop into the help info.
	key := keyName(c.base+"[]", k)
	knownKeys[key] = true

	c.AddChild(key, defValue...)
}

func (c *configSection) AddKnownKey(k string, defValue ...interface{}) {
	key := keyName(c.prefix, k)
	if len(defValue) == 1 {
		c.SetDefault(k, defValue[0])
	} else if len(defValue) > 0 {
		c.SetDefault(k, defValue)
	}
	keysMutex.Lock()
	defer keysMutex.Unlock()
	knownKeys[key] = true

	if c.parent != nil {
		c.parent.AddChild(key, defValue...)
	}
}

func (c *configArray) AddChild(k string, defValue ...interface{}) {
	// When a child is added anywhere below this array, add it to the defaults map
	prefix := c.base + "[]."
	if strings.HasPrefix(k, prefix) {
		c.defaults[strings.TrimPrefix(k, prefix)] = defValue
	}

	// Also bubble it upwards
	if c.parent != nil {
		c.parent.AddChild(k, defValue...)
	}
}

func (c *configSection) AddChild(k string, defValue ...interface{}) {
	// When a child is added anywhere below this key, just bubble it upwards
	if c.parent != nil {
		c.parent.AddChild(k, defValue...)
	}
}

func (c *configSection) SetDefault(k string, defValue interface{}) {
	key := keyName(c.prefix, k)
	viper.SetDefault(key, defValue)
	c.AddChild(key, defValue)
}

func GetConfig() fftypes.JSONObject {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	conf := fftypes.JSONObject{}
	_ = viper.Unmarshal(&conf)
	return conf
}

// GetString gets a configuration string
func GetString(key RootKey) string {
	return root.GetString(string(key))
}
func (c *configSection) GetString(key string) string {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetString(c.prefixKey(key))
}

// GetStringSlice gets a configuration string array
func GetStringSlice(key RootKey) []string {
	return root.GetStringSlice(string(key))
}
func (c *configSection) GetStringSlice(key string) []string {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetStringSlice(c.prefixKey(key))
}

// GetBool gets a configuration bool
func GetBool(key RootKey) bool {
	return root.GetBool(string(key))
}
func (c *configSection) GetBool(key string) bool {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetBool(c.prefixKey(key))
}

// GetDuration gets a configuration time duration with consistent semantics
func GetDuration(key RootKey) time.Duration {
	return root.GetDuration(string(key))
}
func (c *configSection) GetDuration(key string) time.Duration {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return fftypes.ParseToDuration(viper.GetString(c.prefixKey(key)))
}

// GetByteSize get a size in bytes
func GetByteSize(key RootKey) int64 {
	return root.GetByteSize(string(key))
}
func (c *configSection) GetByteSize(key string) int64 {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return fftypes.ParseToByteSize(viper.GetString(c.prefixKey(key)))
}

// GetUint gets a configuration uint
func GetUint(key RootKey) uint {
	return root.GetUint(string(key))
}
func (c *configSection) GetUint(key string) uint {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetUint(c.prefixKey(key))
}

// GetInt gets a configuration uint
func GetInt(key RootKey) int {
	return root.GetInt(string(key))
}
func (c *configSection) GetInt(key string) int {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetInt(c.prefixKey(key))
}

// GetInt64 gets a configuration uint
func GetInt64(key RootKey) int64 {
	return root.GetInt64(string(key))
}
func (c *configSection) GetInt64(key string) int64 {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetInt64(c.prefixKey(key))
}

// GetFloat64 gets a configuration uint
func GetFloat64(key RootKey) float64 {
	return root.GetFloat64(string(key))
}
func (c *configSection) GetFloat64(key string) float64 {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.GetFloat64(c.prefixKey(key))
}

// GetObject gets a configuration map
func GetObject(key RootKey) fftypes.JSONObject {
	return root.GetObject(string(key))
}
func (c *configSection) GetObject(key string) fftypes.JSONObject {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return fftypes.JSONObject(viper.GetStringMap(c.prefixKey(key)))
}

// GetObjectArray gets an array of configuration maps
func GetObjectArray(key RootKey) fftypes.JSONObjectArray {
	return root.GetObjectArray(string(key))
}
func (c *configSection) GetObjectArray(key string) fftypes.JSONObjectArray {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	v, _ := fftypes.ToJSONObjectArray(viper.Get(c.prefixKey(key)))
	return v
}

// Get gets a configuration in raw form
func Get(key RootKey) interface{} {
	return root.Get(string(key))
}
func (c *configSection) Get(key string) interface{} {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return viper.Get(c.prefixKey(key))
}

// Set allows runtime setting of config (used in unit tests)
func Set(key RootKey, value interface{}) {
	root.Set(string(key), value)
}
func (c *configSection) Set(key string, value interface{}) {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	viper.Set(c.prefixKey(key), value)
}

// Resolve gives the fully qualified path of a key
func (c *configSection) Resolve(key string) string {
	keysMutex.Lock()
	defer keysMutex.Unlock()

	return c.prefixKey(key)
}

// SetupLogging initializes logging
func SetupLogging(ctx context.Context) {
	log.SetFormatting(log.Formatting{
		DisableColor:       GetBool(LogNoColor),
		ForceColor:         GetBool(LogForceColor),
		TimestampFormat:    GetString(LogTimeFormat),
		UTC:                GetBool(LogUTC),
		IncludeCodeInfo:    GetBool(LogIncludeCodeInfo),
		JSONEnabled:        GetBool(LogJSONEnabled),
		JSONTimestampField: GetString(LogJSONTimestampField),
		JSONLevelField:     GetString(LogJSONLevelField),
		JSONMessageField:   GetString(LogJSONMessageField),
		JSONFuncField:      GetString(LogJSONFuncField),
		JSONFileField:      GetString(LogJSONFileField),
	})
	logFilename := GetString(LogFilename)
	if logFilename != "" {
		lumberjack := &lumberjack.Logger{
			Filename:   logFilename,
			MaxSize:    int(math.Ceil(float64(GetByteSize(LogFilesize)) / 1024 / 1024)), /* round up in megabytes */
			MaxBackups: GetInt(LogMaxBackups),
			MaxAge:     int(math.Ceil(float64(GetDuration(LogMaxAge)) / float64(time.Hour) / 24)), /* round up in days */
			Compress:   GetBool(LogCompress),
		}
		logrus.SetOutput(lumberjack)
	}
	log.SetLevel(GetString(LogLevel))
	log.L(ctx).Debugf("Log level: %s", logrus.GetLevel())
}

func GenerateConfigMarkdown(ctx context.Context, keys []string) ([]byte, error) {
	b := bytes.NewBuffer([]byte{})

	rootKeyHeaderLevel := 2

	b.WriteString(configDocHeader)

	configObjects := make(map[string][]string)
	configObjectNames := make([]string, 0)

	for _, fullKey := range keys {
		splitKey := strings.Split(fullKey, ".")
		if len(splitKey) > 1 {
			configObjectName := strings.Join(splitKey[:len(splitKey)-1], ".")
			keyName := splitKey[len(splitKey)-1]
			if _, ok := configObjects[configObjectName]; !ok {
				configObjects[configObjectName] = make([]string, 0)
				configObjectNames = append(configObjectNames, configObjectName)
			}
			configObjects[configObjectName] = append(configObjects[configObjectName], keyName)
		}
	}
	sort.Strings(configObjectNames)
	for _, configObjectName := range configObjectNames {
		rowsInTable := []string{}

		sort.Strings(configObjects[configObjectName])
		for _, key := range configObjects[configObjectName] {
			fullKey := fmt.Sprintf("%s.%s", configObjectName, key)
			description, fieldType := getDescriptionForConfigKey(ctx, fullKey)
			if fieldType != i18n.IgnoredType {
				row := fmt.Sprintf("\n|%s|%s|%s|`%v`", key, description, fieldType, Get(RootKey(fullKey)))
				rowsInTable = append(rowsInTable, row)
			}
		}
		if len(rowsInTable) > 0 {
			b.WriteString(fmt.Sprintf("\n\n%s %s", strings.Repeat("#", rootKeyHeaderLevel), configObjectName))
			b.WriteString("\n\n|Key|Description|Type|Default Value|")
			b.WriteString("\n|---|-----------|----|-------------|")
			for _, row := range rowsInTable {
				b.WriteString(row)
			}
		}
	}
	return b.Bytes(), nil
}

func getDescriptionForConfigKey(ctx context.Context, key string) (string, string) {
	configDescriptionKey := "config." + key
	description := i18n.Expand(ctx, i18n.MessageKey(configDescriptionKey))
	fieldType, ok := i18n.GetFieldType(configDescriptionKey)
	if description != configDescriptionKey && ok {
		return description, fieldType
	}
	return getGlobalDescriptionforConfigKey(ctx, key)
}

func getGlobalDescriptionforConfigKey(ctx context.Context, key string) (string, string) {
	// No specific description was found, look for a global
	splitKey := strings.Split(key, ".")
	// Walk through the key structure starting with the most specific key possible, working to the most generic
	for i := 0; i < len(splitKey); i++ {
		configDescriptionKey := "config.global." + strings.Join(splitKey[i:], ".")
		description := i18n.Expand(ctx, i18n.MessageKey(configDescriptionKey))
		fieldType, ok := i18n.GetFieldType(configDescriptionKey)
		if description != configDescriptionKey && ok {
			return description, fieldType
		}
	}
	panic(fmt.Sprintf("Translation for config key '%s' was not found", key))
}

const configDocHeader = `---
layout: default
title: pages.reference
parent: Reference
nav_order: 2
---

# Configuration Reference
{: .no_toc }

<!-- ## Table of contents
{: .no_toc .text-delta }

1. TOC
{:toc} -->

---
`