Merge branch 'unstable'

DeviaVir · Apr 23, 2018 · cf9630e · cf9630e
2 parents 6307098 + 555bd14
commit cf9630e
Show file tree

Hide file tree

Showing 47 changed files with 6,259 additions and 3,299 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 .DS_Store
+.env
 *.tar.gz
 *.log.*
 t.js

diff --git a/README.md b/README.md
@@ -264,7 +264,7 @@ zenbot trade --help
     --asset_capital <amount>          for paper trading, amount of start capital in asset
     --avg_slippage_pct <pct>          avg. amount of slippage to apply to paper trades
     --buy_pct <pct>                   buy with this % of currency balance
-    --buy_max_amt <amt>               buy with up to this amount of currency balance
+    --deposit <amt>                   absolute initial capital (in currency) at the bots disposal (previously --buy_max_amt)
     --sell_pct <pct>                  sell with this % of asset balance
     --markdown_buy_pct <pct>          % to mark down buy price
     --markup_sell_pct <pct>           % to mark up sell price
@@ -284,6 +284,7 @@ zenbot trade --help
     --exact_buy_orders                instead of only adjusting maker buy when the price goes up, adjust it if price has changed at all
     --exact_sell_orders               instead of only adjusting maker sell when the price goes down, adjust it if price has changed at all
     --use_prev_trades                 load and use previous trades for stop-order triggers and loss protection
+    --min_prev_trades                 minimum number of previous trades to load if use_prev_trades is enabled, set to 0 to disable and use trade time instead
     --disable_stats                   disable printing order stats
     --reset_profit                    start new profit calculation from 0
     --use_fee_asset                   Using separated asset to pay for fees. Such as binance's BNB or Huobi's HT

diff --git a/boot.js b/boot.js
@@ -45,7 +45,7 @@ module.exports = function (cb) {
     authStr += '@'
 
     // authMechanism could be a conf.js parameter to support more mongodb authentication methods
-    authMechanism = 'DEFAULT'
+    authMechanism = zenbot.conf.mongo.authMechanism || 'DEFAULT'
   }
 
   if (zenbot.conf.mongo.connectionString) {
@@ -68,4 +68,4 @@ module.exports = function (cb) {
     _.set(zenbot, 'conf.db.mongo', db)
     cb(null, zenbot)
   })
-}
+}
diff --git a/commands/balance.js b/commands/balance.js
@@ -3,8 +3,8 @@ var minimist = require('minimist')
   // eslint-disable-next-line no-unused-vars
   , colors = require('colors')
   , moment = require('moment')
-  , engineFactory = require('../lib/engine')
   , objectifySelector = require('../lib/objectify-selector')
+  , { formatCurrency } = require('../lib/format')
 
 module.exports = function (program, conf) {
   program
@@ -32,14 +32,13 @@ module.exports = function (program, conf) {
       so.selector = s.selector
       so.debug = cmd.debug
       so.mode = 'live'
-      var engine = engineFactory(s, conf)
       function balance () {
         s.exchange.getBalance(s, function (err, balance) {
           if (err) throw err
           s.exchange.getQuote(s, function (err, quote) {
             if (err) throw err
 
-            var bal = moment().format('YYYY-MM-DD HH:mm:ss').grey + ' ' + engine.formatCurrency(quote.ask, true, true, false) + ' ' + (s.product_id).grey + '\n'
+            var bal = moment().format('YYYY-MM-DD HH:mm:ss').grey + ' ' + formatCurrency(quote.ask, s.currency, true, true, false) + ' ' + (s.product_id).grey + '\n'
             bal += moment().format('YYYY-MM-DD HH:mm:ss').grey + ' Asset: '.grey + balance.asset.white + ' Available: '.grey + n(balance.asset).subtract(balance.asset_hold).value().toString().yellow + '\n'
             bal += moment().format('YYYY-MM-DD HH:mm:ss').grey + ' Asset Value: '.grey + n(balance.asset).multiply(quote.ask).value().toString().white + '\n'
             bal += moment().format('YYYY-MM-DD HH:mm:ss').grey + ' Currency: '.grey + balance.currency.white + ' Available: '.grey + n(balance.currency).subtract(balance.currency_hold).value().toString().yellow + '\n'

diff --git a/commands/sim.js b/commands/sim.js
@@ -125,7 +125,7 @@ module.exports = function (program, conf) {
             time: s.period.time
           })
         }
-        s.balance.currency = n(s.balance.currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
+        s.balance.currency = n(s.net_currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
 
         s.balance.asset = 0
         s.lookback.unshift(s.period)

diff --git a/commands/trade.js b/commands/trade.js
@@ -30,7 +30,7 @@ module.exports = function (program, conf) {
     .option('--asset_capital <amount>', 'for paper trading, amount of start capital in asset', Number, conf.asset_capital)
     .option('--avg_slippage_pct <pct>', 'avg. amount of slippage to apply to paper trades', Number, conf.avg_slippage_pct)
     .option('--buy_pct <pct>', 'buy with this % of currency balance', Number, conf.buy_pct)
-    .option('--buy_max_amt <amt>', 'buy with up to this amount of currency balance', Number, conf.buy_max_amt)
+    .option('--deposit <amt>', 'absolute initial capital (in currency) at the bots disposal (previously --buy_max_amt)', Number, conf.deposit)
     .option('--sell_pct <pct>', 'sell with this % of asset balance', Number, conf.sell_pct)
     .option('--markdown_buy_pct <pct>', '% to mark down buy price', Number, conf.markdown_buy_pct)
     .option('--markup_sell_pct <pct>', '% to mark up sell price', Number, conf.markup_sell_pct)
@@ -50,6 +50,7 @@ module.exports = function (program, conf) {
     .option('--exact_buy_orders', 'instead of only adjusting maker buy when the price goes up, adjust it if price has changed at all')
     .option('--exact_sell_orders', 'instead of only adjusting maker sell when the price goes down, adjust it if price has changed at all')
     .option('--use_prev_trades', 'load and use previous trades for stop-order triggers and loss protection')
+    .option('--min_prev_trades <number>', 'minimum number of previous trades to load if use_prev_trades is enabled, set to 0 to disable and use trade time instead', Number, conf.min_prev_trades)
     .option('--disable_stats', 'disable printing order stats')
     .option('--reset_profit', 'start new profit calculation from 0')
     .option('--use_fee_asset', 'Using separated asset to pay for fees. Such as binance\'s BNB or Huobi\'s HT', Boolean, false)
@@ -77,9 +78,14 @@ module.exports = function (program, conf) {
       so.currency_increment = cmd.currency_increment
       so.keep_lookback_periods = cmd.keep_lookback_periods
       so.use_prev_trades = (cmd.use_prev_trades||conf.use_prev_trades)
+      so.min_prev_trades = cmd.min_prev_trades
       so.debug = cmd.debug
       so.stats = !cmd.disable_stats
       so.mode = so.paper ? 'paper' : 'live'
+      if (so.buy_max_amt) {
+        console.log(('--buy_max_amt is deprecated, use --deposit instead!\n').red)
+        so.deposit = so.buy_max_amt
+      }
 
       so.selector = objectifySelector(selector || conf.selector)      
       var engine = engineFactory(s, conf)
@@ -147,7 +153,7 @@ module.exports = function (program, conf) {
 
       /* Implementing statistical Exit */
       function printTrade (quit, dump, statsonly = false) {
-        var tmp_balance = n(s.balance.currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
+        var tmp_balance = n(s.net_currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
         if (quit) {
           if (s.my_trades.length) {
             s.my_trades.push({
@@ -274,7 +280,7 @@ module.exports = function (program, conf) {
         if(!shouldSaveStats) return
 
         var output_lines = []
-        var tmp_balance = n(s.balance.currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
+        var tmp_balance = n(s.net_currency).add(n(s.period.close).multiply(s.balance.asset)).format('0.00000000')
 
         var profit = s.start_capital ? n(tmp_balance).subtract(s.start_capital).divide(s.start_capital) : n(0)
         output_lines.push('last balance: ' + n(tmp_balance).format('0.00000000').yellow + ' (' + profit.format('0.00%') + ')')
@@ -341,7 +347,8 @@ module.exports = function (program, conf) {
 
           //ymd
           var today = dt.slice(2, 4) + dt.slice(5, 7) + dt.slice(8, 10)
-          out_target = so.filename || 'simulations/trade_result_' + so.selector.normalized +'_' + today + '_UTC.html'
+          let out_target_prefix = so.paper ? 'simulations/paper_result_' : 'stats/trade_result_'
+          out_target = so.filename || out_target_prefix + so.selector.normalized +'_' + today + '_UTC.html'
 
           fs.writeFileSync(out_target, out)
           //console.log('\nwrote'.grey, out_target)
@@ -406,10 +413,19 @@ module.exports = function (program, conf) {
           trades.find(opts.query).limit(opts.limit).sort(opts.sort).toArray(function (err, trades) {
             if (err) throw err
             if (trades.length && so.use_prev_trades) {
-              my_trades.find({selector: so.selector.normalized, time : {$gte : trades[0].time}}).limit(0).toArray(function (err, my_prev_trades) {
+              let prevOpts = {
+                query: {
+                  selector: so.selector.normalized
+                },
+                limit: so.min_prev_trades
+              }
+              if (!so.min_prev_trades) {
+                prevOpts.query.time = {$gte : trades[0].time}
+              }
+              my_trades.find(prevOpts.query).sort({$natural:-1}).limit(prevOpts.limit).toArray(function (err, my_prev_trades) {
                 if (err) throw err
                 if (my_prev_trades.length) {
-                  s.my_prev_trades = my_prev_trades.slice(0).sort(function(a,b){return a.time + a.execution_time > b.time + b.execution_time ? -1 : 1}) // simple copy, most recent executed first
+                  s.my_prev_trades = my_prev_trades.reverse().slice(0) // simple copy, less recent executed first
                 }
               })
             }
@@ -441,7 +457,7 @@ module.exports = function (program, conf) {
                   if (err) throw err
                   var prev_session = prev_sessions[0]
                   if (prev_session && !cmd.reset_profit) {
-                    if (prev_session.orig_capital && prev_session.orig_price && ((so.mode === 'paper' && !raw_opts.currency_capital && !raw_opts.asset_capital) || (so.mode === 'live' && prev_session.balance.asset == s.balance.asset && prev_session.balance.currency == s.balance.currency))) {
+                    if (prev_session.orig_capital && prev_session.orig_price && prev_session.deposit === so.deposit && ((so.mode === 'paper' && !raw_opts.currency_capital && !raw_opts.asset_capital) || (so.mode === 'live' && prev_session.balance.asset == s.balance.asset && prev_session.balance.currency == s.balance.currency))) {
                       s.orig_capital = session.orig_capital = prev_session.orig_capital
                       s.orig_price = session.orig_price = prev_session.orig_price
                       if (so.mode === 'paper') {
@@ -553,6 +569,7 @@ module.exports = function (program, conf) {
             session.start_capital = s.start_capital
             session.start_price = s.start_price
             session.num_trades = s.my_trades.length
+            if (so.deposit) session.deposit = so.deposit
             if (!session.orig_capital) session.orig_capital = s.start_capital
             if (!session.orig_price) session.orig_price = s.start_price
             if (s.period) {

diff --git a/conf-sample.js b/conf-sample.js
@@ -14,6 +14,7 @@ c.mongo.username = null
 c.mongo.password = null
 // when using mongodb replication, i.e. when running a mongodb cluster, you can define your replication set here; when you are not using replication (most of the users), just set it to `null` (default).
 c.mongo.replicaSet = null
+c.mongo.authMechanism = null
 
 // default selector. only used if omitting [selector] argument from a command.
 c.selector = 'gdax.BTC-USD'
@@ -58,7 +59,7 @@ c.bittrex.secret = 'YOUR-SECRET'
 c.bitfinex = {}
 c.bitfinex.key = 'YOUR-API-KEY'
 c.bitfinex.secret = 'YOUR-SECRET'
-// May use 'exchange' or 'trading' wallet balances. However margin trading may not work...read the API documentation.
+// May use 'exchange' or 'margin' wallet balances
 c.bitfinex.wallet = 'exchange'
 
 // to enable Bitstamp trading, enter your API credentials:
@@ -170,6 +171,8 @@ c.avg_slippage_pct = 0.045
 c.cancel_after = 'day'
 // load and use previous trades for stop-order triggers and loss protection (live/paper mode only)
 c.use_prev_trades = false
+// minimum number of previous trades to load if use_prev_trades is enabled, set to 0 to disable and use trade time instead
+c.min_prev_trades = 0
 
 // Notifiers:
 c.notifiers = {}

diff --git a/docs/scripts/genetic_backtester.md b/docs/scripts/genetic_backtester.md
@@ -1,6 +1,6 @@
 # Genetic Backtester - Darwin
 
-The Genetic Backtester will execute a range of simulations with a range of parameters, limited by the population size, per strategy. Once all sims in the population are complete, the top results are taken as the starting point for the next generation. This continues indefinitely, until interrupted by the user.
+The Genetic Backtester will execute a range of simulations with a range of parameters, limited by the population size, per strategy. Once all sims in the population are complete, the top results are taken as the starting point for the next generation. This continues indefinitely, until interrupted by the user, or --runGenerations is reached.
 
 ## Installation
 
@@ -38,7 +38,7 @@ The following parameters are available when executing darwin.js:
 --use_strategies="all | strategy1,strategy2"                                            # With this parameter, you can choose to test all, some (comma separated), or just one of the available strategies defined within darwin.
 --population="150"                                                                      # Optional - Number of simulation per generation
 --population_data="./simulations/backtest_[simtimestamp]"                               # Optional - Resume backtesting on a previously terminated backtesting session.
-
+--runGenerations									# Optional - Makes it possible to stop after a number of generations
 ```
 
 ## Results