Merge branch 'merge-upstream'

.gitignore

@@ -8,6 +8,7 @@
 *.gcno
 *.dSYM
 *.rej
+*.pyc
 .cppcheck-suppress
 TAGS
 tags
@@ -34,6 +35,9 @@ gossip_store
 .pytest_cache
 tools/headerversions
 .tmp.lightningrfc/
+contrib/pylightning/build/
+contrib/pylightning/dist/
+contrib/pylightning/pylightning.egg-info/
 
 contrib/compose/groestlcoin_datadir/
 contrib/compose/clightning_groestlcoin_datadir/

.travis.yml

@@ -1,36 +1,42 @@
 language: c
-dist: trusty
-sudo: true
+dist: xenial
+sudo: false
 
 notifications:
   email: false
 
+addons:
+  apt:
+    packages:
+      - python3-pip
+      - libsqlite3-dev
+      - shellcheck
+      - cppcheck
+      - valgrind
+      - libomp-dev
+      - gcc-4.8
+
 env:
-  - ARCH=32 SOURCE_CHECK_ONLY=true
   - ARCH=64 SOURCE_CHECK_ONLY=true CDEBUGFLAGS="-std=gnu11 -g -fstack-protector -O3 -flto" LDFLAGS="-O3 -flto"
-  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
-  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
+#  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
+#  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=clang SOURCE_CHECK_ONLY=false
+  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc-4.8 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
   - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=6 SOURCE_CHECK_ONLY=false
+cache:
+  directories:
+    - dependencies
+    - external
 
-# Trusty (aka 14.04) is way way too old, so run in docker...
 script:
-  - docker pull cdecker/lightning-ci:${ARCH}bit | tee
-  - env | grep -E '^[A-Z_]+\=' | grep -vi travis | tee /tmp/envlist
-  - echo SLOW_MACHINE=1 >> /tmp/envlist
-  - docker run --rm=true --env-file=/tmp/envlist -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit ./configure CC=${COMPILER:-gcc}
-  - $SOURCE_CHECK_ONLY || docker run --rm=true --env-file=/tmp/envlist -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3
-  - $SOURCE_CHECK_ONLY || docker run --rm=true --env-file=/tmp/envlist -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check
-  - (! $SOURCE_CHECK_ONLY) || git clone https://github.com/lightningnetwork/lightning-rfc.git
-  - (! $SOURCE_CHECK_ONLY) || docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make clean
-  - (! $SOURCE_CHECK_ONLY) || docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source BOLTDIR=lightning-rfc
+  .travis/build.sh

.travis/build.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -x
+set -e
+
+CWD=$(pwd)
+export SLOW_MACHINE=1
+export CC=${COMPILER:-gcc}
+export DEVELOPER=${DEVELOPER:-1}
+export SOURCE_CHECK_ONLY=${SOURCE_CHECK_ONLY:-"false"}
+export COMPAT=${COMPAT:-1}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+
+mkdir -p dependencies/bin || true
+
+# Download bitcoind and bitcoin-cli 
+if [ ! -f dependencies/bin/bitcoind ]; then
+    wget https://bitcoin.org/bin/bitcoin-core-0.17.1/bitcoin-0.17.1-x86_64-linux-gnu.tar.gz
+    tar -xzf bitcoin-0.17.1-x86_64-linux-gnu.tar.gz
+    mv bitcoin-0.17.1/bin/* dependencies/bin
+    rm -rf bitcoin-0.17.1-x86_64-linux-gnu.tar.gz bitcoin-0.17.1
+fi
+
+pyenv global 3.7
+pip3 install --user --quiet -r tests/requirements.txt
+pip3 install --quiet \
+     pytest-test-groups==1.0.3
+
+echo "Configuration which is going to be built:"
+echo -en 'travis_fold:start:script.1\\r'
+./configure CC="$CC"
+cat config.vars
+echo -en 'travis_fold:end:script.1\\r'
+
+if [ "$SOURCE_CHECK_ONLY" == "false" ]; then
+    echo -en 'travis_fold:start:script.2\\r'
+    make -j3 > /dev/null
+    echo -en 'travis_fold:end:script.2\\r'
+
+    echo -en 'travis_fold:start:script.3\\r'
+    make check
+    echo -en 'travis_fold:end:script.3\\r'
+else
+    git clone https://github.com/lightningnetwork/lightning-rfc.git
+    make check-source BOLTDIR=lightning-rfc
+fi

CHANGELOG.md

@@ -1,39 +1,84 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
-and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.7.0] - 2019-02-28: "Actually an Altcoin"
 
+This release named by Mark Beckwith @wythe.
 
 ### Added
 
 - plugins: fully enabled, and ready for you to write some!
 - plugins: `pay` is now a plugin.
-- lightning-cli: `help <cmd>` finds man pages even if `make install` not run.
+- protocol: `pay` will now use routehints in invoices if it needs to.
+- build: reproducible source zipfile and Ubuntu 18.04.1 build.
+- JSON API: New command `paystatus` gives detailed information on `pay` commands.
+- JSON API: `getroute`, `invoice`, `sendpay` and `pay` commands `msatoshi`
+  parameter can have suffixes `msat`, `sat` (optionally with 3 decimals) or `grs`
+  (with 1 to 11 decimal places).
+- JSON API: `fundchannel` and `withdraw` commands `satoshi`
+  parameter can have suffixes `msat` (must end in `000`), `sat` or `grs`
+  (with 1 to 8 decimal places).
+- JSON API: `decodepay`, `getroute`, `sendpay`, `pay`, `listpeers`, `listfunds`, `listchannels` and
+  all invoice commands now return an `amount_msat` field which has an `msat` suffix.
+- JSON API: `listfunds` `channels` now has `_msat` fields for each existing raw amount field, with `msat` suffix.
 - JSON API: `waitsendpay` now has an `erring_direction` field.
 - JSON API: `listpeers` now has a `direction` field in `channels`.
 - JSON API: `listchannels` now takes a `source` option to filter by node id.
-- JSON API: New command `paystatus` gives detailed information on `pay` commands.
+- JSON API: `getroute` `riskfactor` argument is simplified; `pay` now defaults to setting it to 10.
+- JSON API: `sendpay` now takes a `bolt11` field, and it's returned in `listpayments` and `waitsendpay`.
+- JSON API: `fundchannel` and `withdraw` now have a new parameter `minconf` that limits coinselection to outputs that have at least `minconf` confirmations (default 1). (#2380)
+- JSON API: `listfunds` now displays addresses for all outputs owned by the wallet (#2387)
+- JSON API: `waitsendpay` and `sendpay` output field `label` as specified by `sendpay` call.
+- JSON API: `listpays` command for higher-level payment view than `listpayments`, especially important with multi-part-payments coming.
+- JSON API: `listpayments` is now `listsendpays`.
+- lightning-cli: `help <cmd>` finds man pages even if `make install` not run.
+- pylightning: New class 'Millisatoshi' can be used for JSON API, and new '_msat' fields are turned into this on reading.
 
 ### Changed
 
-- The `short_channel_id` separator has been changed to be `x` to match the specification.
+- protocol: `option_data_loss_protect` is now enabled by default.
+- JSON API: The `short_channel_id` separator has been changed to be `x` to match the specification.
 - JSON API: `listpeers` now includes `funding_allocation_msat`, which returns a map of the amounts initially funded to the channel by each peer, indexed by channel id.
-- `option_data_loss_protect` is now enabled by default.
+- JSON API: `help` with a `command` argument gives a JSON array, like other commands.
+- JSON API: `sendpay` `description` parameter is renamed `label`.
+- JSON API: `pay` now takes an optional `label` parameter for labelling payments, in place of never-used `description`.
+- build: we'll use the system libbase58 and libsodium if found suitable.
 
 ### Deprecated
 
 Note: You should always set `allow-deprecated-apis=false` to test for
 changes.
 
+We recommend that you transition to the reading the new JSON `_msat`
+fields for your own sanity checking, and that you similarly
+provide appropriate suffixes for JSON input fields.
+
+- JSON API: `short_channel_id` fields in JSON commands with `:` separators (use `x` instead).
+- JSON API: `pay` `description` is deprecated, as is support for BOLT11 strings using `h`.
+- JSON API: `sendpay` parameter `description` and `waitsendpay` and `sendpay` output fields `description` (now `label`).
+- JSON API: `listpayments` has been deprecated (you probably want `listpays`)
+
 ### Removed
 
+- JSON API: the `waitsendpay` command error return no longer includes `channel_update`
+
 ### Fixed
 
 - Protocol: handling `query_channel_range` for large numbers of blocks
   (eg. 4 billion) was slow due to a bug.
+- Fixed occasional deadlock with peers when exchanging huge amounts of gossip.
+- Fixed a crash when running in daemon-mode due to db filename overrun (#2348)
+- Handle lnd sending premature 'funding_locked' message when we're expected 'reestablish';
+  we used to close channel if this happened.
+- Cleanup peers that started opening a channel, but then disconnected. These
+  would leave a dangling entry in the DB that would cause this peer to be
+  unable to connect. (PR #2371)
+- You can no longer make giant unpayable "wumbo" invoices.
+- CLTV of total route now correctly evaluated when finding best route.
+- `riskfactor` arguments to `pay` and `getroute` now have an effect.
 
 ### Security
 
@@ -79,7 +124,7 @@ changes.
 - Protocol: don't send invalid onion errors if peer says onion was bad.
 - Protocol: don't crash when peer sends a 0-block-expiry HTLC.
 - pylightning: handle multiple simultanous RPC replies reliably.
-
+- build: we use `--prefix` as handed to `./configure`
 
 ### Security
 
@@ -267,7 +312,7 @@ There predate the BOLT specifications, and are only of vague historic interest:
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
-[Unreleased]: https://github.com/ElementsProject/lightning/compare/v0.6.3...HEAD
+[0.7.0]: https://github.com/ElementsProject/lightning/compare/v0.6.3...HEAD
 [0.6.3]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.3
 [0.6.2]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.2
 [0.6.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.1

Dockerfile

@@ -55,7 +55,8 @@ VOLUME [ "/root/.lightning" ]
 
 COPY --from=builder /opt/lightningd/cli/lightning-cli /usr/bin
 COPY --from=builder /opt/lightningd/lightningd/lightning* /usr/bin/
-COPY --from=builder /opt/groestlcoin /usr/bin
+COPY --from=builder /opt/lightningd/plugins/pay /usr/libexec/c-lightning/plugins/
+COPY --from=builder /opt/groestlcoin/bin /usr/bin
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

Makefile

@@ -24,6 +24,12 @@ VG=valgrind -q --error-exitcode=7
 VG_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all
 endif
 
+ifneq ($(ASAN),0)
+SANITIZER_FLAGS=-fsanitize=address
+else
+SANITIZER_FLAGS=
+endif
+
 ifeq ($(DEVELOPER),1)
 DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1
 else
@@ -46,7 +52,7 @@ endif
 
 # Timeout shortly before the 600 second travis silence timeout
 # (method=thread to support xdist)
-PYTEST_OPTS := -v --timeout=550 --timeout_method=thread
+PYTEST_OPTS := -v --timeout=550 --timeout_method=thread -p no:logging
 
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
@@ -90,6 +96,7 @@ CCAN_OBJS :=					\
 	ccan-str-base32.o			\
 	ccan-str-hex.o				\
 	ccan-str.o				\
+	ccan-strmap.o				\
 	ccan-take.o				\
 	ccan-tal-grab_file.o			\
 	ccan-tal-link.o				\
@@ -179,13 +186,13 @@ ALL_PROGRAMS =
 
 CPPFLAGS = -DBINTOPKGLIBEXECDIR='"'$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))'"'
 CWARNFLAGS := -Werror -Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition
-CDEBUGFLAGS := -std=gnu11 -g -fstack-protector
+CDEBUGFLAGS := -std=gnu11 -g -fstack-protector $(SANITIZER_FLAGS)
 CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS)
 
 # We can get configurator to run a different compile cmd to cross-configure.
 CONFIGURATOR_CC := $(CC)
 
-LDFLAGS = $(PIE_LDFLAGS)
+LDFLAGS = $(PIE_LDFLAGS) $(SANITIZER_FLAGS)
 ifeq ($(STATIC),1)
 LDLIBS = -L/usr/local/lib -Wl,-dn -lgmp -lsqlite3 -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
@@ -295,6 +302,8 @@ check-python:
 	@# W503: line break before binary operator
 	@flake8 --ignore=E501,E731,W503 --exclude=contrib/pylightning/lightning/__init__.py ${PYSRC}
 
+	PYTHONPATH=contrib/pylightning:$$PYTHONPATH $(PYTEST) contrib/pylightning/
+
 check-includes:
 	@tools/check-includes.sh
 
@@ -303,7 +312,7 @@ check-includes:
 	@git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs grep -n 'list_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
 
 check-cppcheck: .cppcheck-suppress
-	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck -q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress
+	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck -q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
 
 check-shellcheck:
 	@git ls-files -- "*.sh" | xargs shellcheck
@@ -317,7 +326,13 @@ check-tmpctx:
 check-discouraged-functions:
 	@if git grep -E "[^a-z_/](fgets|fputs|gets|scanf|sprintf)\(" -- "*.c" "*.h" ":(exclude)ccan/"; then exit 1; fi
 
-check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions
+# Don't access amount_msat and amount_sat members directly without a good reason
+# since it risks overflow.
+check-amount-access:
+	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
+
+check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
 full-check: check check-source
 
@@ -421,13 +436,12 @@ unittest/%: %
 	$(VG) $(VG_TEST_ARGS) $* > /dev/null
 
 # Installation directories
-prefix = /usr/local
-exec_prefix = $(prefix)
+exec_prefix = $(PREFIX)
 bindir = $(exec_prefix)/bin
 libexecdir = $(exec_prefix)/libexec
 pkglibexecdir = $(libexecdir)/$(PKGNAME)
 plugindir = $(pkglibexecdir)/plugins
-datadir = $(prefix)/share
+datadir = $(PREFIX)/share
 docdir = $(datadir)/doc/$(PKGNAME)
 mandir = $(datadir)/man
 man1dir = $(mandir)/man1
@@ -475,7 +489,7 @@ PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_openingd
 PLUGINS=plugins/pay
 
-install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS)
+install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)

README.md

@@ -16,6 +16,7 @@ http://lightning.network.
 [![Build Status][travis-ci]][travis-ci-link]
 [![Pull Requests Welcome][prs]][prs-link]
 [![Irc][IRC]][IRC-link]
+[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)](https://lightning.readthedocs.io/)
 
 [travis-ci]: https://travis-ci.org/ElementsProject/lightning.svg?branch=master
 [travis-ci-link]: https://travis-ci.org/ElementsProject/lightning
@@ -58,7 +59,7 @@ For the impatient here's the gist of it for Ubuntu and Debian:
     sudo apt-get update
     sudo apt-get install -y \
       autoconf automake build-essential git libtool libgmp-dev \
-      libsqlite3-dev python python3 net-tools zlib1g-dev
+      libsqlite3-dev python python3 net-tools zlib1g-dev libsodium-dev
     git clone https://github.com/Groestlcoin/lightning.git
     cd lightning
     ./configure
@@ -99,18 +100,6 @@ open a channel:
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-If you don't have any testcoins you can get a few from a faucet such as
-[Kiwi's testnet faucet][kiw].
-You can send it directly to the `lightningd` address.
-
-You may need to generate a p2sh-segwit address if the faucet does not support
-bech32:
-
-    # Return a p2sh-segwit address
-    cli/lightning-cli newaddr p2sh-segwit
-
-[kiw]: https://testnet.manu.backend.hamburg/faucet
-
 Confirm `lightningd` got funds by:
 
     # Returns an array of on-chain funds.
@@ -225,7 +214,7 @@ JSON-RPC interface is documented in the following manual pages:
 * [getroute](doc/lightning-getroute.7.txt)
 * [sendpay](doc/lightning-sendpay.7.txt)
 * [pay](doc/lightning-pay.7.txt)
-* [listpayments](doc/lightning-listpayments.7.txt)
+* [listpays](doc/lightning-listpays.7.txt)
 * [decodepay](doc/lightning-decodepay.7.txt)
 
 For simple access to the JSON-RPC interface you can use the