Rename app, add docs, make more robust (#36)

* chore: rename app, add docs, make more robust

* fix: missing references

* docs: typo [skip ci]

* docs: README update [skip ci]

* docs: README update [skip ci]

* docs: options listed in README, advanced use case section [skip ci]

* docs: typo [skip ci]

* chore: increase verbosity

* feat: add install.sh

Dockerfile

@@ -3,15 +3,15 @@ FROM golang:1.19 as app-builder
 ARG DESCRIBE=""
 WORKDIR /go/src/app
 COPY . .
-RUN make clean linux MULTIARCH=true && ls -ali ./release/balance-agent*linux*
+RUN make clean linux MULTIARCH=true && ls -ali ./release/bolt-agent*linux*
 
 FROM scratch
 
 ENV AWS_DEFAULT_REGION us-east-1
 
 COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=app-builder /go/src/app/release/balance-agent*linux* /balance-agent
+COPY --from=app-builder /go/src/app/release/bolt-agent*linux* /bolt-agent
 VOLUME ["/tmp"]
 
 USER 666
-ENTRYPOINT ["/balance-agent"]
+ENTRYPOINT ["/bolt-agent"]

README.md

@@ -4,63 +4,113 @@ Agent is a piece of software that is able to run on lightning node (we provide r
 
 ## Contents:
 
-Currently we have:
+### bolt-agent
 
-### balance-agent
-
-utility to monitor channel balances
+utility to monitor and manage lightning node
 
 The usage should be pretty self expanatory:
 
 ```
 NAME:
-   balance-agent - Utility to monitor channel balances
+   bolt-agent - Utility to monitor and manage lightning node
 
 USAGE:
-   balance-agent [global options] command [command options] [arguments...]
+   bolt-agent [global options] command [command options] [arguments...]
 
 VERSION:
-   v0.0.41
+   v0.0.46
 
 COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
    --allowedentropy value      allowed entropy in bits for channel balances (default: 64)
    --apikey value              api key
-   --channel-whitelist value   Path to file containing a whitelist of channels
+   --channel-whitelist value   path to file containing a whitelist of channels
    --interval value            interval to poll - 10s, 1m, 10m or 1h (default: "10s")
    --lnddir value              path to lnd's base directory (default: "/home/user/.lnd")
    --macaroonpath value        path to macaroon file
    --private                   report private data as well (default: false)
-   --preferipv4                If you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)
+   --preferipv4                if you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)
    --rpcserver value           host:port of ln daemon (default: "localhost:10009")
    --tlscertpath value         path to TLS certificate (default: "/home/user/.lnd/tls.cert")
-   --channel-whitelist value   Path to file containing a whitelist of channels
+   --channel-whitelist value   path to file containing a whitelist of channels
    --verbosity value           log level for V logs (default: 0)
    --help, -h                  show help
    --version, -v               print the version
+
+ADVANCED OPTIONS:
+   --datastore-url             datastore URL (default: agent-api.bolt.observer:443)
+   --ignorecln                 ignore CLN socket (default: false)
+   --keepalive value           keepalive interval (if nothing changed after this time an empty message will be sent) (default: "60s")
+   --public value              used together with --channel-whitelist (default: false)
+   --url value                 report url (default: "https://ingress.bolt.observer/api/node-data-report/")
+   --userest                   use REST API when true instead of gRPC (default: false)
+   --fetch-invoices            fetch invoices (default: 0)
+   --fetch-forwards            fetch forwards (default: 0)
+   --fetch-payments            fetch payments (default: 0)
+
+GLOG OPTIONS (described in https://github.com/google/glog#setting-flags):
+   --verbosity (instead of -v)
+   --logtostderr
+   --stderrthreshold
+   --alsologtostderr
+   --vmodule
+   --log_dir
+   --log_backtrace_at
 ```
 
-It tries the best to have sane defaults so you can just start it up on your node without further hassle.
-If you have a custom lnd dir (`/storage` in the example) you might need:
+Agent works with LND and CoreLightning and tries the best to have sane defaults so you can just start it up on your node without further hassle.
+
+Example:
 ```
-balance-agent --lnddir /storage/lnd/ --tlscertpath /storage/lnd/data/secrets/lnd.cert
+bolt-agent --apikey key
 ```
-but that should be it.
 
-Only thing that is mandatory is the apikey which can also be provided through `API_KEY` environment variable.
+Only thing that is mandatory is the apikey which can also be provided through `API_KEY` environment variable:
+```
+export API_KEY=changeme
+bolt-agent
+```
+
+Agent is using `glog` library for logging which needs writable `/tmp` (use `--logtostderr` to prevent writing to filesystem).
+
+Flag `--private` means whether you want to report data for private ("unannounced") channels too (when true). The default is false.
+There is an advanced option of filtering what channels exactly are reported documented in "Filtering on agent side" section.
+
+For more information about what exactly is reported read [DATA.md](./DATA.md).
+
+#### Advanced options
+
+If you have a custom LND directory (`/storage` in the example) you might need:
+
+```
+bolt-agent --lnddir /storage/lnd/ --tlscertpath /storage/lnd/data/secrets/lnd.cert
+```
+
+but that should be it.
 
 By default it will try to communicate with LND using local gRPC connection and `readonly.macaroon`.
 
-You can use `--userest` to start using REST API (default is as mentioned gRPC) - in that case `--rpcserver` can be complete URL to the endpoint.
+You can use `--userest` to start using REST API - in that case `--rpcserver` can be complete URL to the endpoint.
 
-Of course you can also start the agent on a remote node and just point it to the correct endpoint, but since this is not the default
-use cases flags `--userest` and `--rpcserver` are not advertied in help.
+```
+bolt-agent --userest --rpcserver https://bolt.observer/endpoint
+```
 
-Flag `--private` means whether you want to report data for private ("unannounced") channels too (when true). The default is false.
+For CoreLightning it will use UNIX domain socket. By default agent will try to use `~/.lightning/bitcoin/lightning-rpc`.
+If that file exists program will default to CoreLightning so if have LND installed on the node too and do not want to use CLN there is a flag
+`--ignorecln`. You can manually override name of UNIX domain socket using `--rpcserver` flag.
+Example:
 
-For more information about what exactly is reported read [DATA.md](./DATA.md).
+```
+bolt-agent --rpcserver /storage/cln/socket
+```
+
+Make sure agent is running with correct permissions to access the socket.
+
+There is support for remote connections via commando plugin but you cannot use that option via command-line flags at the moment. We suggest to use
+`socat` utility if you wish to monitor a remote CLN node.
 
 ## Install
 
@@ -72,14 +122,14 @@ Installation steps:
 * fetch latest revision from https://github.com/bolt-observer/agent/releases
 
 ```
-wget https://github.com/bolt-observer/agent/releases/download/v0.0.41/balance-agent-v0.0.41-linux.zip https://github.com/bolt-observer/agent/releases/download/v0.0.41/manifest-v0.0.41.txt.asc https://github.com/bolt-observer/agent/releases/download/v0.0.41/manifest-v0.0.41.txt
+wget https://github.com/bolt-observer/agent/releases/download/v0.0.46/bolt-agent-v0.0.46-linux.zip https://github.com/bolt-observer/agent/releases/download/v0.0.46/manifest-v0.0.46.txt.asc https://github.com/bolt-observer/agent/releases/download/v0.0.46/manifest-v0.0.46.txt
 ```
 
 * verify integrity
 
 ```
 wget -qO- https://raw.githubusercontent.com/bolt-observer/agent/main/scripts/keys/fiksn.asc | gpg --import
-gpg --verify manifest-v0.0.41.txt.asc manifest-v0.0.41.txt
+gpg --verify manifest-v0.0.46.txt.asc manifest-v0.0.46.txt
 ```
 
 and you should see:
@@ -91,32 +141,32 @@ gpg: Good signature from "Gregor Pogacnik <[email protected]>" [ultimate]
 * unpack the compressed binary
 
 ```
-unzip balance-agent-v0.0.41-linux.zip
+unzip bolt-agent-v0.0.46-linux.zip
 ```
 
 * copy the binary to a common place
 
 ```
-cp balance-agent-v0.0.41-linux /usr/local/bin/balance-agent
+cp bolt-agent-v0.0.46-linux /usr/local/bin/bolt-agent
 ```
 
 * start the binary
 
 ```
-balance-agent -apikey changeme
+bolt-agent -apikey changeme
 ```
 
-* you may want to run the agent as a systemd service (we have a simple systemd template [here](./balance-agent.service))
+* you may want to run the agent as a systemd service (we have a simple systemd template [here](./bolt-agent.service))
 
 You need to do that only once:
 
 ```
-wget https://raw.githubusercontent.com/bolt-observer/agent/main/balance-agent.service
-cp balance-agent.service /etc/systemd/system/balance-agent.service
-$EDITOR /etc/systemd/system/balance-agent.service # to change API_KEY
+wget https://raw.githubusercontent.com/bolt-observer/agent/main/bolt-agent.service
+cp bolt-agent.service /etc/systemd/system/bolt-agent.service
+$EDITOR /etc/systemd/system/bolt-agent.service # to change API_KEY
 systemctl daemon-reload
-systemctl enable balance-agent
-systemctl start balance-agent
+systemctl enable bolt-agent
+systemctl start bolt-agent
 ```
 
 ## Docker
@@ -126,12 +176,13 @@ You can use the docker image from GitHub:
 Usage:
 
 ```
-docker run -v /tmp:/tmp -e API_KEY=changeme ghcr.io/bolt-observer/agent:v0.0.41
+docker run -v /tmp:/tmp -e API_KEY=changeme ghcr.io/bolt-observer/agent:v0.0.46
 ```
 
 ## Filtering on agent side
 
 You can limit what channnels are reported using `--channel-whitelist` option. It specifies a local file path to be used as a whitelist of what channels to report.
+Channel IDs are short channel identifiers in LND format (a valid id is `792080480214450176` but not `720393x2228x0`)
 When adding `--private` to `--channel-whitelist` this means every private channel AND whatever is listed in the file. There is also `--public` to allow all public channels.
 Using the options without `--channel-whitelist` makes no sense since by default all public channels are reported however it has to be explicit with `--channel-whitelist` in order
 to automatically allow all public channels (beside what is allowed through the file).
@@ -161,7 +212,7 @@ whatever
 Internally we use:
 * [nodedata](./nodedata): an abstraction for running the periodic checks and reporting balance related changes
 * [filter](./filter): this is used to filter specific channels on the agent side
-* [checkermonitoring](./checkermonitoring): is used for reporting metrics via Graphite (not used directly in balance-agent here)
+* [checkermonitoring](./checkermonitoring): is used for reporting metrics via Graphite (not used directly in bolt-agent here)
 * [lightning](./lightning): an abstraction around lightning node API (that furthermore heavily depends on common code from [lnd](https://github.com/lightningnetwork/lnd))
 * [agent](./agent): is the GRPC client for new funcionality
 * [raw](./raw): is the connection between [agent](./agent) API and [lightning](./lightning) API

balance-agent.service → bolt-agent.service

@@ -1,10 +1,10 @@
-# /etc/systemd/system/balance-agent.service
+# /etc/systemd/system/bolt-agent.service
 [Unit]
-Description=BalanceAgent service
+Description=BoltObserverAgent service
 
 [Service]
 Environment="API_KEY=changeme"
-ExecStart=/usr/local/bin/balance-agent
+ExecStart=/usr/local/bin/bolt-agent
 
 Type=simple
 Restart=on-failure

cmd/balance-agent/main.go → cmd/bolt-agent/main.go

@@ -218,7 +218,7 @@ func getApp() *cli.App {
 		},
 		&cli.StringFlag{
 			Name:   whitelist,
-			Usage:  "Path to file containing a whitelist of channels",
+			Usage:  "path to file containing a whitelist of channels",
 			Hidden: false,
 		},
 		&cli.StringFlag{
@@ -246,7 +246,7 @@ func getApp() *cli.App {
 		},
 		&cli.BoolFlag{
 			Name:  "preferipv4",
-			Usage: "If you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)",
+			Usage: "if you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)",
 		},
 		&cli.StringFlag{
 			Name:  "rpcserver",
@@ -260,12 +260,12 @@ func getApp() *cli.App {
 		},
 		&cli.BoolFlag{
 			Name:   "userest",
-			Usage:  "Use REST API when true instead of gRPC",
+			Usage:  "use REST API when true instead of gRPC",
 			Hidden: true,
 		},
 		&cli.DurationFlag{
 			Name:   "keepalive",
-			Usage:  "Keepalive interval (if nothing changed after this time an empty message will be sent)",
+			Usage:  "keepalive interval (if nothing changed after this time an empty message will be sent)",
 			Value:  60 * time.Second,
 			Hidden: true,
 		},
@@ -276,13 +276,13 @@ func getApp() *cli.App {
 		},
 		&cli.StringFlag{
 			Name:   "url",
-			Usage:  "Report URL",
+			Usage:  "report URL",
 			Value:  "https://ingress.bolt.observer/api/node-data-report/",
 			Hidden: true,
 		},
 		&cli.StringFlag{
 			Name:   "uniqueid",
-			Usage:  "Unique identifier",
+			Usage:  "unique identifier",
 			Value:  "",
 			Hidden: true,
 		},
@@ -308,25 +308,25 @@ func getApp() *cli.App {
 		},
 		&cli.StringFlag{
 			Name:   "datastore-url",
-			Usage:  "Datastore URL",
+			Usage:  "datastore URL",
 			Value:  "agent-api.bolt.observer:443",
 			Hidden: true,
 		},
 		&cli.Int64Flag{
 			Name:   "fetch-invoices",
-			Usage:  "Fetch invoices",
+			Usage:  "fetch invoices",
 			Value:  0,
 			Hidden: true,
 		},
 		&cli.Int64Flag{
 			Name:   "fetch-forwards",
-			Usage:  "Fetch forwards",
+			Usage:  "fetch forwards",
 			Value:  0,
 			Hidden: true,
 		},
 		&cli.Int64Flag{
 			Name:   "fetch-payments",
-			Usage:  "Fetch payments",
+			Usage:  "fetch payments",
 			Value:  0,
 			Hidden: true,
 		},
@@ -639,11 +639,7 @@ func sender(ctx context.Context, cmdCtx *cli.Context, apiKey string) {
 }
 
 func senderWithRetries(ctx context.Context, cmdCtx *cli.Context, apiKey string) error {
-	lightningAPI := mkGetLndAPI(cmdCtx)()
-	if lightningAPI == nil {
-		return fmt.Errorf("get GRPC ligtning failure")
-	}
-	sender, err := raw.MakeSender(ctx, apiKey, cmdCtx.String("datastore-url"), lightningAPI)
+	sender, err := raw.MakeSender(ctx, apiKey, cmdCtx.String("datastore-url"), mkGetLndAPI(cmdCtx))
 	if err != nil {
 		return fmt.Errorf("get GRPC fetcher failure %v", err)
 	}
@@ -671,8 +667,8 @@ func senderWithRetries(ctx context.Context, cmdCtx *cli.Context, apiKey string)
 
 func main() {
 	app := getApp()
-	app.Name = "balance-agent"
-	app.Usage = "Utility to monitor channel balances"
+	app.Name = "bolt-agent"
+	app.Usage = "Utility to monitor and manage lightning node"
 	app.Action = func(cmdCtx *cli.Context) error {
 		glogShim(cmdCtx)
 		if err := nodeDataChecker(cmdCtx); err != nil {