Enable new configuration approaches

Enable using a configuration file provided as a command line option,
environment variables for specific fields and a configuration file read
from a path given as an environment variable.

README.md

@@ -18,8 +18,48 @@ yarn test
 
 To run:
 ```sh
-cp config.yaml.template config.yaml
-# Modify the configuration
-vim config.yaml
-yarn start -c config.yaml
+# Some configuration must be given, see below.
+yarn start
 ```
+
+### Configuration
+
+[The included configuration file template](./config.yaml) contains all of the intended configuration options. The absolute minimum fields that must be provided are marked with a comment `# Required` above the name of the field. They may be provided as environment variables, as well.
+
+Configuration priority, higher overrides lower:
+1. A configuration file whose path is given as a command line option.
+1. Environment variables except for `CONFIG_PATH`.
+1. A configuration file whose path is given by the environment variable `CONFIG_PATH`. If a configuration file has been specified via a command line option, `CONFIG_PATH` is ignored.
+
+E.g. `yarn start -c config.yaml` excludes any bessersmith-specific environment variables from being used.
+
+Some configuration must be given, none is hardcoded. A configuration file is not strictly necessary.
+
+A handy configuration pattern for deployment:
+1. Copy `config.yaml` and replace the dummy values, especially the secrets like ClientIds, usernames and passwords.
+1. Deliver the new configuration file into Docker Swarm as [a secret](https://docs.docker.com/engine/swarm/secrets/).
+1. Point `CONFIG_PATH` to the secret. Do not use the command line option `-c`.
+1. Override the non-secret values with the other environment variables as usual in the CI/CD pipeline.
+
+The list of environment variables used by bessersmith:
+- `CONFIG_PATH`: The path to the configuration file.
+- `BUNYAN_NAME`: The value for the field `name` in [bunyan](https://github.com/trentm/node-bunyan) log messages.
+- `BUNYAN_LEVEL`: The logging level used by bunyan. Currently one of `fatal`, `error`, `warn`, `info`, `debug` or `trace`.
+- `MQTT_SUB_URL`: The URL of the MQTT broker used for subscribing. The string is given to the `connect` function in [MQTT.js](https://github.com/mqttjs/MQTT.js). Currently one of the following protocols must be used: `mqtt`, `mqtts`, `tcp`, `tls`, `ws` or `wss`.
+- `MQTT_SUB_PORT`: The port for the MQTT broker used for subscribing.
+- `MQTT_SUB_CLIENT_ID`: The ClientId for the subscribing connection. Make sure this differs from all other ClientIds, e.g. by using a random suffix.
+- `MQTT_SUB_CLEAN`: A boolean on whether to use a clean session for subscribing. With `false` bessersmith asks for a persistent session.
+- `MQTT_SUB_USERNAME`: The username for the subscribing connection.
+- `MQTT_SUB_PASSWORD`: The passowrd for the subscribing connection.
+- `MQTT_SUB_QOS`: The Quality of Service level that bessersmith should subscribe with.
+- `MQTT_SUB_TOPIC`: The topic that bessersmith should subscribe to to get the custom MONO JSON messages.
+- `MQTT_PUB_URL`: The URL of the MQTT broker used for publishing. Running a broker on `localhost` may get useful.
+- `MQTT_PUB_PORT`: The port for the MQTT broker used for publishing.
+- `MQTT_PUB_CLIENT_ID`: The ClientId for the publishing connection. Make sure this differs from all other ClientIds, as well.
+- `MQTT_PUB_CLEAN`: A boolean on whether to use a clean session for publishing.
+- `MQTT_PUB_USERNAME`: The username for the publishing connection.
+- `MQTT_PUB_PASSWORD`: The passowrd for the publishing connection.
+- `MQTT_PUB_QOS`: The Quality of Service level that bessersmith should publish with.
+- `MQTT_PUB_TOPIC`: The topic into which bessersmith should publish GTFS Realtime TripUpdates.
+- `CACHE_TTL_IN_SECONDS`: The time to live in seconds for any entry in the trip [cache](https://github.com/ptarjan/node-cache).
+- `PROTO_PATH`: The path to the GTFS Realtime protocol buffer schema file.

index.js

@@ -1,3 +1,4 @@
+const _ = require("lodash");
 const assert = require("assert");
 const fs = require("fs");
 const neodoc = require("neodoc");
@@ -9,22 +10,92 @@ const help = `
 bessersmith
 
 Usage:
-  bessersmith -c <CONFIG_YAML>
+  bessersmith -c <CONFIG_PATH>
   bessersmith -h | --help
 
 Options:
-  -c --config=<CONFIG_YAML> Specify a YAML configuration file to use.
+  -c --config=<CONFIG_PATH> Specify the path to the UTF-8 YAML configuration
+                            file.
   -h --help                 Show this screen.
   --version                 Show version.
 `;
 
+const constructConfigFromEnvironment = () => {
+  const filled = {
+    bunyan: {
+      name: process.env.BUNYAN_NAME,
+      level: process.env.BUNYAN_LEVEL
+    },
+    mqtt: {
+      subscribe: {
+        url: process.env.MQTT_SUB_URL,
+        connectionOptions: {
+          port: process.env.MQTT_SUB_PORT,
+          clientId: process.env.MQTT_SUB_CLIENT_ID,
+          clean: process.env.MQTT_SUB_CLEAN,
+          username: process.env.MQTT_SUB_USERNAME,
+          password: process.env.MQTT_SUB_PASSWORD
+        },
+        subscriptionOptions: {
+          qos: process.env.MQTT_SUB_QOS
+        },
+        topic: process.env.MQTT_SUB_TOPIC
+      },
+      publish: {
+        url: process.env.MQTT_PUB_URL,
+        connectionOptions: {
+          port: process.env.MQTT_PUB_PORT,
+          clientId: process.env.MQTT_PUB_CLIENT_ID,
+          clean: process.env.MQTT_PUB_CLEAN,
+          username: process.env.MQTT_PUB_USERNAME,
+          password: process.env.MQTT_PUB_PASSWORD
+        },
+        publishingOptions: {
+          qos: process.env.MQTT_PUB_QOS
+        },
+        topic: process.env.MQTT_PUB_TOPIC
+      },
+      cache: {
+        ttlInSeconds: process.env.CACHE_TTL_IN_SECONDS
+      },
+      protoPath: process.env.PROTO_PATH
+    }
+  };
+  return _.omitBy(filled, _.isUndefined);
+};
+
+const assertFieldExists = (value, name) => {
+  if (typeof value === "undefined") {
+    assert.fail(`${name} must be provided`);
+  }
+};
+
+const getConfig = cliConfigPath => {
+  const useCliConfigPath = typeof cliConfigPath === "undefined";
+  const configPath = cliConfigPath || process.env.CONFIG_PATH;
+  let configFromFile = {};
+  if (typeof configPath !== "undefined") {
+    configFromFile = yaml.safeLoad(fs.readFileSync(configPath, "utf8"));
+  }
+  const envConfig = constructConfigFromEnvironment();
+  let config = {};
+  if (!useCliConfigPath) {
+    config = _.merge(envConfig, configFromFile);
+  } else {
+    config = _.merge(configFromFile, envConfig);
+  }
+
+  assertFieldExists(config.bunyan.name, "bunyan.name");
+  assertFieldExists(config.mqtt.subscribe.topic, "mqtt.subscribe.topic");
+  assertFieldExists(config.mqtt.publish.topic, "mqtt.publish.topic");
+  assertFieldExists(config.protoPath, "protoPath");
+
+  return config;
+};
+
 const main = () => {
   const args = neodoc.run(help);
-  const configFilename = args["--config"] || process.env.CONFIG_YAML;
-  if (typeof configFilename === "undefined") {
-    assert.fail("Configuration file path was not given");
-  }
-  const config = yaml.safeLoad(fs.readFileSync(configFilename, "utf8"));
+  const config = getConfig(args["--config"]);
   Promise.resolve(run(config));
 };