New Feature: Adds Data Directory Support (#1012)

- Updates the go-algorand submodule hash to point to rel/beta
- Moves the cpu profiling file, pid file and indexer configuration file
  to be options of only the daemon sub-command
- Changes os.Exit() to be a panic with a special handler.  This is so
  that defer's are handled instead of being ignored.
- Detects auto-loading configuration files in the data directory and
  issues errors if equivalent command line arguments are supplied.
- Updates the README with instructions on how to use the auto-loading
  configuration files and the data directory.

Makefile

@@ -34,8 +34,8 @@ idb/postgres/internal/schema/setup_postgres_sql.go:	idb/postgres/internal/schema
 	cd idb/postgres/internal/schema && go generate
 
 idb/mocks/IndexerDb.go:	idb/idb.go
-	go install github.com/vektra/mockery/.../
-	cd idb && mockery -name=IndexerDb
+	go install github.com/vektra/mockery/v2@latest
+	cd idb && mockery --name=IndexerDb
 
 # check that all packages (except tests) compile
 check: go-algorand

README.md

@@ -195,6 +195,7 @@ Settings can be provided from the command line, a configuration file, or an envi
 | default-balances-limit        |         | default-balances-limit        | INDEXER_DEFAULT_BALANCES_LIMIT        |
 | max-applications-limit        |         | max-applications-limit        | INDEXER_MAX_APPLICATIONS_LIMIT        |
 | default-applications-limit    |         | default-applications-limit    | INDEXER_DEFAULT_APPLICATIONS_LIMIT    |
+| data-dir                      | i       | data-dir                      | INDEXER_DATA                      |
 
 ## Command line
 
@@ -204,33 +205,23 @@ The command line arguments always take priority over the config file and environ
 ~$ ./algorand-indexer daemon --pidfile /var/lib/algorand/algorand-indexer.pid --algod /var/lib/algorand --postgres "host=mydb.mycloud.com user=postgres password=password dbname=mainnet"`
 ```
 
-## Configuration file
-Default values are placed in the configuration file. They can be overridden with environment variables and command line arguments.
+## Data Directory
 
-The configuration file must named **indexer**, **indexer.yml**, or **indexer.yaml**. The filepath may be set on the CLI using `--configfile` or `-c`.
-When the filepath is not provided on the CLI, it must also be in the correct location. Only one configuration file is loaded, the path is searched in the following order:
-* `./` (current working directory)
-* `$HOME`
-* `$HOME/.algorand-indexer`
-* `$HOME/.config/algorand-indexer`
-* `/etc/algorand-indexer/`
+The Indexer data directory is the location where the Indexer can store and/or load data needed for runtime operation and configuration.
 
-Here is an example **indexer.yml** file:
-```
-postgres-connection-string: "host=mydb.mycloud.com user=postgres password=password dbname=mainnet"
-pidfile: "/var/lib/algorand/algorand-indexer.pid"
-algod-data-dir: "/var/lib/algorand"
-```
+**It is a required argument for Indexer daemon operation. Supply it to the Indexer via the `--data-dir` flag.**
 
-If it is in the current working directory along with the indexer command we can start the indexer daemon with:
-```
-~$ ./algorand-indexer daemon
-```
 
-If it is not in the current working directory along with the indexer command we can start the indexer daemon with:
-```
-~$ ./algorand-indexer daemon -c <full-file-location>/indexer.yml
-```
+### Auto-Loading Configuration
+
+The Indexer will scan the data directory at startup and load certain configuration files if they are present.  The files are as follows:
+
+- `indexer.yml` - Indexer Configuration File
+- `api_config.yml` - API Parameter Enable/Disable Configuration File
+- 
+**NOTE:** It is not allowed to supply both the command line flag AND have an auto-loading configuration file in the data directory.  Doing so will result in an error.
+
+To see an example of how to use the data directory to load a configuration file check out the [Disabling Parameters Guide](docs/DisablingParametersGuide.md).
 
 ## Example environment variable
 
@@ -244,6 +235,25 @@ The same indexer configuration from earlier can be made in bash with the followi
 ~$ ./algorand-indexer daemon
 ```
 
+
+## Configuration file
+Default values are placed in the configuration file. They can be overridden with environment variables and command line arguments.
+
+The configuration file must named **indexer.yml** and placed in the data directory (see above). The filepath may be set on the CLI using `--configfile` or `-c` but this functionality is deprecated.
+
+Here is an example **indexer.yml** file:
+```
+postgres-connection-string: "host=mydb.mycloud.com user=postgres password=password dbname=mainnet"
+pidfile: "/var/lib/algorand/algorand-indexer.pid"
+algod-data-dir: "/var/lib/algorand"
+```
+
+Place this file in the data directory (`/tmp/data-dir` in this example) and supply it to the Indexer daemon:
+```
+~$ ./algorand-indexer daemon --data-dir /tmp/data-dir
+```
+
+
 # Systemd
 
 `/lib/systemd/system/algorand-indexer.service` can be partially overridden by creating `/etc/systemd/system/algorand-indexer.service.d/local.conf`. The most common things to override will be the command line and pidfile. The overriding local.conf file might be this:

cmd/algorand-indexer/api_config.go

@@ -26,21 +26,21 @@ var apiConfigCmd = &cobra.Command{
 		err = configureLogger()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 		swag, err := generated.GetSwagger()
 
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to get swagger: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 
 		options := makeOptions()
 		if suppliedAPIConfigFile != "" {
 			potentialDisabledMapConfig, err := api.MakeDisabledMapConfigFromFile(swag, suppliedAPIConfigFile)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "failed to created disabled map config from file: %v", err)
-				os.Exit(1)
+				panic(exit{1})
 			}
 			options.DisabledMapConfig = potentialDisabledMapConfig
 		}
@@ -57,11 +57,11 @@ var apiConfigCmd = &cobra.Command{
 
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to output yaml: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 
 		fmt.Fprint(os.Stdout, output)
-		os.Exit(0)
+		panic(exit{0})
 
 	},
 }

cmd/algorand-indexer/daemon.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"runtime/pprof"
 	"strings"
 	"sync"
 	"syscall"
@@ -14,6 +16,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/algorand/go-algorand/rpcs"
+	"github.com/algorand/go-algorand/util"
 	"github.com/algorand/indexer/api"
 	"github.com/algorand/indexer/api/generated/v2"
 	"github.com/algorand/indexer/config"
@@ -49,50 +52,129 @@ var (
 	defaultApplicationsLimit  uint32
 	enableAllParameters       bool
 	indexerDataDir            string
+	cpuProfile                string
+	pidFilePath               string
+	configFile                string
 )
 
 var daemonCmd = &cobra.Command{
 	Use:   "daemon",
 	Short: "run indexer daemon",
 	Long:  "run indexer daemon. Serve api on HTTP.",
-	//Args:
 	Run: func(cmd *cobra.Command, args []string) {
 		var err error
 		config.BindFlags(cmd)
 		err = configureLogger()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err)
-			os.Exit(1)
+			panic(exit{1})
+		}
+
+		if indexerDataDir == "" {
+			fmt.Fprint(os.Stderr, "indexer data directory was not provided")
+			panic(exit{1})
+		}
+
+		// Create the directory if it doesn't exist
+		if _, err := os.Stat(indexerDataDir); os.IsNotExist(err) {
+			err := os.Mkdir(indexerDataDir, 0755)
+			maybeFail(err, "failure while creating data directory: %v", err)
+		}
+
+		// Detect the various auto-loading configs from data directory
+		indexerConfigFound := util.FileExists(filepath.Join(indexerDataDir, autoLoadIndexerConfigName))
+		paramConfigFound := util.FileExists(filepath.Join(indexerDataDir, autoLoadParameterConfigName))
+
+		// If we auto-loaded configs but a user supplied them as well, we have an error
+		if indexerConfigFound {
+			if configFile != "" {
+				logger.Errorf(
+					"indexer configuration was found in data directory (%s) as well as supplied via command line.  Only provide one.",
+					filepath.Join(indexerDataDir, autoLoadIndexerConfigName))
+				panic(exit{1})
+			}
+			// No config file supplied via command line, auto-load it
+			configs, err := os.Open(configFile)
+			if err != nil {
+				maybeFail(err, "%v", err)
+			}
+			defer configs.Close()
+			err = viper.ReadConfig(configs)
+			if err != nil {
+				maybeFail(err, "invalid config file (%s): %v", viper.ConfigFileUsed(), err)
+			}
+		}
+
+		if paramConfigFound {
+			if suppliedAPIConfigFile != "" {
+				logger.Errorf(
+					"api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one.",
+					filepath.Join(indexerDataDir, autoLoadParameterConfigName))
+				panic(exit{1})
+			}
+			suppliedAPIConfigFile = filepath.Join(indexerDataDir, autoLoadParameterConfigName)
+			fmt.Printf("Auto-loading parameter configuration file: %s", suppliedAPIConfigFile)
+
+		}
+
+		if pidFilePath != "" {
+			fmt.Printf("Creating PID file at: %s\n", pidFilePath)
+			fout, err := os.Create(pidFilePath)
+			maybeFail(err, "%s: could not create pid file, %v", pidFilePath, err)
+			_, err = fmt.Fprintf(fout, "%d", os.Getpid())
+			maybeFail(err, "%s: could not write pid file, %v", pidFilePath, err)
+			err = fout.Close()
+			maybeFail(err, "%s: could not close pid file, %v", pidFilePath, err)
+			defer func(name string) {
+				err := os.Remove(name)
+				if err != nil {
+					logger.WithError(err).Errorf("%s: could not remove pid file", pidFilePath)
+				}
+			}(pidFilePath)
+		}
+
+		if cpuProfile != "" {
+			var err error
+			profFile, err = os.Create(cpuProfile)
+			maybeFail(err, "%s: create, %v", cpuProfile, err)
+			defer profFile.Close()
+			err = pprof.StartCPUProfile(profFile)
+			maybeFail(err, "%s: start pprof, %v", cpuProfile, err)
+			defer pprof.StopCPUProfile()
+		}
+
+		if configFile != "" {
+			configs, err := os.Open(configFile)
+			if err != nil {
+				maybeFail(err, "%v", err)
+			}
+			defer configs.Close()
+			err = viper.ReadConfig(configs)
+			if err != nil {
+				maybeFail(err, "invalid config file (%s): %v", viper.ConfigFileUsed(), err)
+			}
+			fmt.Printf("Using configuration file: %s\n", configFile)
 		}
 
 		// If someone supplied a configuration file but also said to enable all parameters,
 		// that's an error
 		if suppliedAPIConfigFile != "" && enableAllParameters {
 			fmt.Fprint(os.Stderr, "not allowed to supply an api config file and enable all parameters")
-			os.Exit(1)
+			panic(exit{1})
 		}
 
 		if algodDataDir == "" {
 			algodDataDir = os.Getenv("ALGORAND_DATA")
 		}
 
-		if indexerDataDir == "" {
-			indexerDataDir = os.Getenv("INDEXER_DATA")
-		}
-
-		if indexerDataDir != "" {
-			if _, err := os.Stat(indexerDataDir); os.IsNotExist(err) {
-				err := os.Mkdir(indexerDataDir, 0755)
-				maybeFail(err, "indexer data directory error, %v", err)
-			}
-		}
-
 		ctx, cf := context.WithCancel(context.Background())
 		defer cf()
 		{
 			cancelCh := make(chan os.Signal, 1)
 			signal.Notify(cancelCh, syscall.SIGTERM, syscall.SIGINT)
 			go func() {
+				// Need to redefine exitHandler() for every go-routine
+				defer exitHandler()
 				<-cancelCh
 				logger.Println("Stopping Indexer.")
 				cf()
@@ -123,8 +205,14 @@ var daemonCmd = &cobra.Command{
 		defer db.Close()
 		var wg sync.WaitGroup
 		if bot != nil {
+			if indexerDataDir == "" {
+				fmt.Fprint(os.Stderr, "missing indexer data directory")
+				panic(exit{1})
+			}
 			wg.Add(1)
 			go func() {
+				// Need to redefine exitHandler() for every go-routine
+				defer exitHandler()
 				defer wg.Done()
 
 				// Wait until the database is available.
@@ -150,7 +238,7 @@ var daemonCmd = &cobra.Command{
 					// If context is not expired.
 					if ctx.Err() == nil {
 						logger.WithError(err).Errorf("fetcher exited with error")
-						os.Exit(1)
+						panic(exit{1})
 					}
 				}
 			}()
@@ -199,10 +287,15 @@ func init() {
 
 	daemonCmd.Flags().StringVarP(&indexerDataDir, "data-dir", "i", "", "path to indexer data dir, or $INDEXER_DATA")
 
+	daemonCmd.Flags().StringVarP(&cpuProfile, "cpuprofile", "", "", "file to record cpu profile to")
+	daemonCmd.Flags().StringVarP(&pidFilePath, "pidfile", "", "", "file to write daemon's process id to")
+	daemonCmd.Flags().StringVarP(&configFile, "configfile", "c", "", "file path to configuration file (indexer.yml)")
+
 	viper.RegisterAlias("algod", "algod-data-dir")
 	viper.RegisterAlias("algod-net", "algod-address")
 	viper.RegisterAlias("server", "server-address")
 	viper.RegisterAlias("token", "api-token")
+	viper.RegisterAlias("data-dir", "data")
 }
 
 // makeOptions converts CLI options to server options
@@ -248,14 +341,14 @@ func makeOptions() (options api.ExtraOptions) {
 		swag, err := generated.GetSwagger()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to get swagger: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 
 		logger.Infof("supplied api configuration file located at: %s", suppliedAPIConfigFile)
 		potentialDisabledMapConfig, err := api.MakeDisabledMapConfigFromFile(swag, suppliedAPIConfigFile)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to created disabled map config from file: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 		options.DisabledMapConfig = potentialDisabledMapConfig
 	}

cmd/algorand-indexer/import.go

@@ -20,7 +20,7 @@ var importCmd = &cobra.Command{
 		err := configureLogger()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err)
-			os.Exit(1)
+			panic(exit{1})
 		}
 
 		db, availableCh := indexerDbFromFlags(idb.IndexerDbOptions{})