Merge branch 'dev'

README.md

@@ -20,6 +20,8 @@ You can install and run Lightning Jet in [Docker](#docker) (for advanced users w
 
 Jet is available on [EmbassyOS](https://github.com/Start9Labs/embassy-os) and can rebalance channels on [Start9](https://start9.com/latest/) products that run the OS.
 
+Jet can be installed in [Ubuntu VM](#ubuntu-vm) on Windows, Mac OS, and other platforms.
+
 ## Prerequisites
 
 Make sure to [install node](https://nodejs.org/en/download/) if you don't have it already. Run `node -v` to check if you have `node` and whether it is up to date (version 16.x+). Update `node` in case of an old version (this will also update `npm`).  
@@ -38,7 +40,7 @@ Make sure `npm` is up to date (version 8.x) by running `npm -v`. Update `npm` in
 ```bash
 git clone https://github.com/itsneski/lightning-jet
 cd lightning-jet
-npm install --build-from-source --python=/usr/bin/python3
+npm install
 nano ./api/config.json
 ```
 Edit `config.json`: set correct absolute (not relative) paths for `macaroonPath` and `tlsCertPath`. On umbrel, macaroons are typically located at `/home/umbrel/umbrel/lnd/data/chain/bitcoin/mainnet/admin.macaroon`, tls cert is at `/home/umbrel/umbrel/lnd/tls.cert`.
@@ -102,6 +104,7 @@ jet help
 |`jet monitor`|Monitors ongoing rebalances, rebalance history, and stuck htlcs. Warns about the state of BOLT database (channel.db); for example, jet will warn when the channel.db grows over a threshold.|
 |`jet monitor --status`|Monitors the status of rebalances; shows whether rebalances are paused or active; provides recommendation for local ppm range.|
 |`jet stats`|Displays profitability metrics over a time period, including delta(s) with previous time period. Node operators can use this tool to A/B test new channels and fee updates on existing channels.|
+|`jet probes`|Displays nodes (discovered during probes) that have signaled a commitment to liquidity. This tool can be used to identify prospects for new channels.|
 |`jet htlc-analyzer`|Analyzes failed htlcs and lists peers sorted based on missed routing opportunities. Missed routing opportunities are typically due to [outbound] peers not having sufficient liquidity and/or having low fees.|
 |`jet htlc-analyzer ln2me --hours 12`|Shows missed routing opportunities for ln2me node over the past 12 hours.|
 |`jet analyze-fees`|Analyzes fees for [outbound] peers and recommends whether to increase or decrease fees based on routing history.|
@@ -165,6 +168,24 @@ Prepend [all commands](#how-to-run) with `docker exec -it lightning-jet`:
 docker exec -it lightning-jet jet help
 ```
 
+## Ubuntu VM
+
+Lightning Jet can be installed in Ubuntu VM on Windows, Mac OS, and other platforms.
+
+1. [Install VirtualBox and set up Ubuntu image](https://ubuntu.com/tutorials/how-to-run-ubuntu-desktop-on-a-virtual-machine-using-virtualbox#1-overview)
+
+2. [Add user to the sudo group](https://www.tecmint.com/fix-user-is-not-in-the-sudoers-file-the-incident-will-be-reported-ubuntu/)
+
+3. Install curl: `sudo snap install curl`
+
+4. [Install node](https://github.com/nodesource/distributions/blob/master/README.md#using-ubuntu)
+
+5. Install git: `sudo apt install git`
+
+6. [Install jet](#installation)
+
+7. Copy over admin.macaroon (and tls.cert if needed) as in [Voltage Cloud](#voltage-cloud)
+
 ## Voltage Cloud
 
 Lightning Jet can rebalance your node in Voltage Cloud by connecting to it remotely via a secure grpc connection. You can select any platform (host-based or cloud, like AWS) to install Jet as long as it supports [Node](https://nodejs.org/en/download/). Jet runs as a daemon (background process), so its best to select a platform that supports running Jet 24/7.

api/constants.js

@@ -6,6 +6,8 @@ exports.logNameRegEx = /[^a-zA-Z0-9.]/g;
 exports.telegramBotHelpPage = 'https://github.com/itsneski/lightning-jet#telegram-bot';
 exports.maxTxnInterval = 30 * 24; // hours
 exports.defaultTxnInterval = 7 * 24; // hours
+exports.defaultProbeInterval = 7 * 24; // hours
+exports.defaultProbeTopN = 10;
 
 // monitor
 exports.monitor = {

api/rebalance.js

@@ -32,6 +32,7 @@ const {recordRebalanceAvoid} = require('../db/utils');
 const {listRebalanceAvoidSync} = require('../db/utils');
 const {recordActiveRebalanceSync} = require('../db/utils');
 const {deleteActiveRebalanceSync} = require('../db/utils');
+const {recordLiquidity} = require('../db/utils');
 const {listPeersMapSync} = require('../lnd-api/utils');
 const {getNodeFeeSync} = require('../lnd-api/utils');
 const {rebalanceSync} = require('../bos/rebalance');
@@ -197,7 +198,8 @@ module.exports = ({from, to, amount, ppm = config.rebalancer.maxPpm || constants
       console.log('* time left:', timeLeft, 'mins');
 
       // call bos rebalance; async logger will be notified about route evals and log messages
-      let lastRoute; // last route that was evaluated
+      let lastRoute;  // last route that was evaluated
+      let lastAmount; // last amount that was evaluated
       const rebalanceLogger = {
         eval: (route) => {
           console.log('\nprobing route:', route);
@@ -221,6 +223,10 @@ module.exports = ({from, to, amount, ppm = config.rebalancer.maxPpm || constants
             }
           })
         },
+        amount: (amount) => {
+          console.log('evaluating amount:', amount);
+          lastAmount = amount;
+        },
         debug: (msg) => {
           if (config.debugMode) console.log('bos rebalance debug:', stringify(msg));
         },
@@ -271,6 +277,11 @@ module.exports = ({from, to, amount, ppm = config.rebalancer.maxPpm || constants
           if (lastRoute) {
             let nodes = lastRoute;
             if (nodes) {
+              // record nodes in the db for further analysis
+              nodes.forEach(n => {
+                recordLiquidity({node: n.id, sats: lastAmount, ppm: n.ppm});
+              })
+
               // find a node with max ppm that's not already on the avoid list
               // its enough to select one node to unblock the route
               let max;
@@ -434,6 +445,13 @@ module.exports = ({from, to, amount, ppm = config.rebalancer.maxPpm || constants
           // record result in the db for further optimation
           recordRebalance(iterationStart, outId, inId, AMOUNT, amount, Math.round(1000000 * fees / amount), type);
 
+          // record nodes on the route for future analysis
+          if (lastRoute) {  // shouldn't be empty but just in case
+            lastRoute.forEach(n => {
+              recordLiquidity({node: n.id, sats: amount, ppm: n.ppm});
+            })
+          }
+
           console.log('* total amount rebalanced:', numberWithCommas(amountRebalanced));
           if (fees > 0) {
             console.log('* fees spent:', fees);
@@ -501,7 +519,7 @@ module.exports = ({from, to, amount, ppm = config.rebalancer.maxPpm || constants
     let id;
     Object.values(peerMap).forEach(p => {
       if (p.name.toLowerCase().indexOf(str.toLowerCase()) >= 0) {
-        if (id) throw new Error('more than one pub id associated with ' + str);
+        if (id) throw new Error('more than one peer associated with ' + str + '; narrow your selection');
         id = p.id;
       }
     })

bos/rebalance.js

@@ -58,6 +58,8 @@ module.exports = {
     const mylogger = {
       debug: (msg) => { return args.logger.debug(debug) },
       info: (msg) => {
+        if (msg && msg.evaluating_amount !== undefined) return args.logger.amount(msg.evaluating_amount);
+
         try {
           let nodes = parseNodes(msg);
           if (nodes) return args.logger.eval(nodes);

db/utils.js

@@ -22,6 +22,7 @@ const FEE_HISTORY_TABLE = 'fee_history';
 const ACTIVE_REBALANCE_TABLE = 'active_rebalance';
 const CHANNEL_EVENTS_TABLE = 'channel_events';
 const TXN_TABLE = 'txn';
+const LIQUIDITY_TABLE = 'liquidity';  // lists peers that were ready to commit liquidity for payments
 
 var testMode = false;
 
@@ -38,6 +39,49 @@ const uniqueArr = arr => arr.filter(function(elem, pos) {
 })
 
 module.exports = {
+  reportLiquidity(fromDate, toDate) {
+    const db = getHandle();
+    let list = [];
+
+    // get forwards and rebalances
+    try {
+      let done, error;
+      db.serialize(() => {
+        let q = 'SELECT node, COUNT() as count, SUM(sats) as sats_sum, ROUND(avg(ppm)) as avg_ppm, MIN(ppm) as min_ppm, MAX(ppm) as max_ppm FROM liquidity';
+        if (fromDate) q += ' WHERE date >= ' + fromDate;
+        if (toDate) q += ' AND date < ' + toDate;
+        q += ' GROUP BY node ORDER BY count DESC';
+        if (testMode) console.log(q);
+        db.each(q, (err, row) => {
+          list.push(row);
+        }, (err) => {
+          error = err;
+          done = true;
+        })
+      })
+      deasync.loopWhile(() => !done);
+      if (error) console.error('reportLiquidity:', error.message);
+    } catch(err) {
+      console.error('reportLiquidity:', err.message);
+    } finally {
+      closeHandle(db);
+    }
+    return list;
+  },
+  recordLiquidity({node, sats, ppm}) {
+    const pref = 'recordLiquidity:';
+    if (!node || !sats || ppm === undefined) throw new Error('missing params');
+
+    let db = getHandle();
+    const vals = constructInsertString([Date.now(), node, sats, ppm]);
+    const cols = '(date, node, sats, ppm)';
+    let cmd = 'INSERT INTO ' + LIQUIDITY_TABLE + ' ' + cols + ' VALUES (' + vals + ')';
+    // record async, no need to wait for completion
+    executeDb(db, cmd, (err) => {
+      if (err) console.error(pref, err.message);
+      closeHandle(db);
+    })
+  },
   txnReset() {
     let db = getHandle();
     db.serialize(() => {
@@ -406,28 +450,29 @@ module.exports = {
     }
   },
   fetchTelegramMessageSync() {
+    const pref = 'fetchTelegramMessageSync:';
     let db = getHandle();
+    let done, error;
+    let messages = [];
     try {
-      let done;
-      let messages = [];
       db.serialize(function() {
         let q = 'SELECT rowid, * FROM ' + TELEGRAM_MESSAGES_TABLE + ' ORDER BY date ASC';
-        db.each(q, function(err, row) {
+        db.each(q, (err, row) => {
           messages.push({id:row.rowid, message:row.message});
-        }, function(error) {
-          if (error) throw new Error(error.message);
+        }, (err) => {
+          error = err;
           done = true;
         })
       })
-      while(done === undefined) {
-        require('deasync').runLoopOnce();
-      }
-      return messages;
-    } catch(error) {
-      console.error('fetchTelegramMessageSync:', error.message);
+    } catch(err) {
+      error = err;
+      done = true;
     } finally {
       closeHandle(db);
     }
+    deasync.loopWhile(() => !done);
+    if (error) console.error(pref, error.message);
+    return messages;
   },
   recordTelegramMessageSync(msg) {
     let db = getHandle();
@@ -766,10 +811,8 @@ module.exports = {
 }
 
 function getHandle() {
-  let handle;
-  if (testMode) handle = new sqlite3.Database(testDbFile); 
-  else handle = new sqlite3.Database(dbFile);
-  return handle;
+  if (testMode) return new sqlite3.Database(testDbFile); 
+  else return new sqlite3.Database(dbFile);
 }
 
 function closeHandle(handle) {
@@ -829,10 +872,15 @@ function createTables() {
     createActiveRebalanceTable(db);
     createChannelEventsTable(db);
     createTxnTable(db);
+    createLiquidityTable(db);
   })
   closeHandle(db);
 }
 
+function createLiquidityTable(db) {
+  executeDbSync(db, "CREATE TABLE IF NOT EXISTS " + LIQUIDITY_TABLE + " (date INTEGER NOT NULL, chan TEXT, node TEXT NOT NULL, sats INTEGER NOT NULL, ppm INTEGER NOT NULL)");
+}
+
 function createTxnTable(db) {
   // type: 'forward' or 'payment'
   executeDbSync(db, "CREATE TABLE IF NOT EXISTS " + TXN_TABLE + " (date INTEGER NOT NULL, txdate_ns INTEGER NOT NULL, digest TEXT NOT NULL UNIQUE, type TEXT NOT NULL, from_chan INTEGER NOT NULL, to_chan INTEGER NOT NULL, amount INTEGER NOT NULL, fee INTEGER NOT NULL)");