Merge remote-tracking branch 'upstream/master'

CHANGELOG.md

@@ -7,6 +7,40 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 TODO: Insert version codename, and username of the contributor that named the release.
 -->
 
+## [22.11.1] - 2022-12-09: "Alameda Yield Generator II"
+
+### Added
+
+ - JSON-RPC: reverts requirement for "jsonrpc" "2.0" inside requests (still deprecated though, just for a while longer!) ([#5783])
+
+### Changed
+
+ - config: `announce-addr-dns` needs to be set to *true* to put DNS names into node announcements, otherwise they are suppressed.
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - config: `announce-addr-dns` (currently defaults to `false`).  This will default to `true` once enough of the network has upgraded to understand DNS entries. ([#5796])
+
+### Fixed
+
+ - Build: arm32 compiler error in fetchinvoice, due to bad types on 32-bit platforms. ([#5785])
+ - JSON-RPC: `autoclean-once` response `uncleaned` count is now correct. ([#5775])
+ - Plugin: `autoclean` could misperform or get killed due to lightningd's invalid handling of JSON batching. ([#5775])
+ - reckless verbosity properly applied. ([#5781])
+ - wireaddr: #5657 allow '_' underscore in hostname part of DNS FQDN ([#5789])
+
+[#5781]: https://github.com/ElementsProject/lightning/pull/5781
+[#5783]: https://github.com/ElementsProject/lightning/pull/5783
+[#5775]: https://github.com/ElementsProject/lightning/pull/5775
+[#5789]: https://github.com/ElementsProject/lightning/pull/5789
+[#5796]: https://github.com/ElementsProject/lightning/pull/5796
+[#5785]: https://github.com/ElementsProject/lightning/pull/5785
+[#5775]: https://github.com/ElementsProject/lightning/pull/5775
+[22.11.1]: https://github.com/ElementsProject/lightning/releases/tag/v22.11.1
+
+
 ## [22.11] - 2022-11-30: "Alameda Yield Generator"
 
 

common/bolt12.c

@@ -432,7 +432,7 @@ bool bolt12_has_prefix(const char *str)
 }
 
 /* Inclusive span of tlv range >= minfield and <= maxfield */
-size_t tlv_span(const u8 *tlvstream, size_t minfield, size_t maxfield,
+size_t tlv_span(const u8 *tlvstream, u64 minfield, u64 maxfield,
 		size_t *startp)
 {
 	const u8 *cursor = tlvstream;

common/bolt12.h

@@ -135,7 +135,7 @@ bool bolt12_has_prefix(const char *str);
  *
  * Returns length, so 0 means nothing found.
 */
-size_t tlv_span(const u8 *tlvstream, size_t minfield, size_t maxfield,
+size_t tlv_span(const u8 *tlvstream, u64 minfield, u64 maxfield,
 		size_t *start);
 
 /* Get offer_id referred to by various structures. */

common/test/run-ip_port_parsing.c

@@ -126,11 +126,16 @@ int main(int argc, char *argv[])
 	assert(is_dnsaddr("123example.com"));
 	assert(is_dnsaddr("example123.com"));
 	assert(is_dnsaddr("is-valid.3hostname123.com"));
-	assert(!is_dnsaddr("UPPERCASE.invalid.com"));
+	assert(is_dnsaddr("just-a-hostname-with-dashes"));
+	assert(is_dnsaddr("lightningd_dest.underscore.allowed.in.hostname.part.com"));
+	assert(is_dnsaddr("UpperCase.valiD.COM"));
+	assert(is_dnsaddr("punycode.xn--bcher-kva.valid.com"));
+	assert(!is_dnsaddr("nonpunycode.bücher.invalid.com"));
 	assert(!is_dnsaddr("-.invalid.com"));
 	assert(!is_dnsaddr("invalid.-example.com"));
 	assert(!is_dnsaddr("invalid.example-.com"));
 	assert(!is_dnsaddr("invalid..example.com"));
+	assert(!is_dnsaddr("underscore.not.allowed.in.domain_name.com"));
 
 	/* Grossly invalid. */
 	assert(!separate_address_and_port(tmpctx, "[", &ip, &port));

common/wireaddr.c

@@ -375,18 +375,23 @@ bool is_wildcardaddr(const char *arg)
 	return streq(arg, "");
 }
 
-/* Rules:
+/* The rules to check for DNS FQDNs, see `man 7 hostname`
  *
  * - not longer than 255
- * - segments are separated with . dot
- * - segments do not start or end with - hyphen
- * - segments must be longer thant zero
- * - lowercase a-z and digits 0-9 and - hyphen
+ * - labels are separated with . dot
+ * - labels do not start or end with - hyphen
+ * - labels must be longer than zero
+ * - allow ASCII letters a-z, A-Z, digits 0-9 and - hyphen
+ * - additionally we allow for an '_' underscore in the first hostname label
+ * - other characters must be punycoded rfc3492
+ *
+ * See `man 7 hostname` and https://www.rfc-editor.org/rfc/rfc1035
  */
 bool is_dnsaddr(const char *arg)
 {
 	size_t i, arglen;
 	int lastdot;
+	int numlabels;
 
 	if (is_ipaddr(arg) || is_toraddr(arg) || is_wildcardaddr(arg))
 		return false;
@@ -396,8 +401,10 @@ bool is_dnsaddr(const char *arg)
 	if (arglen > 255)
 		return false;
 	lastdot = -1;
+	numlabels = 0;
 	for (i = 0; i < arglen; i++) {
 		if (arg[i] == '.') {
+			numlabels++;
 			/* segment must be longer than zero */
 			if (i - lastdot == 1)
 				return false;
@@ -412,10 +419,15 @@ bool is_dnsaddr(const char *arg)
 			return false;
 		if (arg[i] >= 'a' && arg[i] <= 'z')
 			continue;
+		if (arg[i] >= 'A' && arg[i] <= 'Z')
+			continue;
 		if (arg[i] >= '0' && arg[i] <= '9')
 			continue;
 		if (arg[i] == '-')
 			continue;
+		/* allow for _ underscores in the first hostname part */
+		if (arg[i] == '_' && numlabels == 0)
+			continue;
 		return false;
 	}
 	return true;

contrib/docker/linuxarm32v7.Dockerfile

@@ -5,7 +5,7 @@
 # * final: Copy the binaries required at runtime
 # The resulting image uploaded to dockerhub will only contain what is needed for runtime.
 # From the root of the repository, run "docker build -t yourimage:yourtag -f contrib/linuxarm32v7.Dockerfile ."
-FROM debian:buster-slim as downloader
+FROM debian:bullseye-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -18,7 +18,7 @@ RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.
     && echo "01b54b934d5f5deb32aa4eb4b0f71d0e76324f4f0237cc262d59376bf2bdc269 /opt/tini" | sha256sum -c - \
     && chmod +x /opt/tini
 
-ENV GROESTLCOIN_VERSION 2.20.1
+ENV GROESTLCOIN_VERSION 22.0
 ENV GROESTLCOIN_TARBALL groestlcoin-${GROESTLCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 ENV GROESTLCOIN_URL https://github.com/Groestlcoin/groestlcoin/releases/download/v$GROESTLCOIN_VERSION/$GROESTLCOIN_TARBALL
 ENV GROESTLCOIN_ASC_URL https://github.com/Groestlcoin/groestlcoin/releases/download/v$GROESTLCOIN_VERSION/SHA256SUMS.asc
@@ -40,11 +40,36 @@ RUN mkdir /opt/groestlcoin && cd /opt/groestlcoin \
     && tar -xzvf $GROESTLCOIN_TARBALL $BD/groestlcoin-cli --strip-components=1 \
     && rm $GROESTLCOIN_TARBALL
 
-FROM debian:buster-slim as builder
+FROM debian:bullseye-slim as builder
 
 ENV LIGHTNINGD_VERSION=master
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates autoconf automake build-essential gettext git libtool python3 python3-pip python3-setuptools python3-mako wget gnupg dirmngr git lowdown \
-  libc6-armhf-cross gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
+
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        autoconf \
+        automake \
+        build-essential \
+        ca-certificates \
+        curl \
+        dirmngr \
+        gettext \
+        git \
+        gnupg \
+        libpq-dev \
+        libtool \
+        libffi-dev \
+        python3 \
+        python3-dev \
+        python3-mako \
+        python3-pip \
+        python3-venv \
+        python3-setuptools \
+        wget && \
+    # arm32v7 compilers
+    apt-get install -qq -y --no-install-recommends \
+        libc6-armhf-cross \
+        gcc-arm-linux-gnueabihf \
+        g++-arm-linux-gnueabihf
 
 ENV target_host=arm-linux-gnueabihf
 
@@ -57,12 +82,12 @@ STRIP=${target_host}-strip \
 QEMU_LD_PREFIX=/usr/${target_host} \
 HOST=${target_host}
 
-RUN wget -q https://zlib.net/zlib-1.2.12.tar.gz \
-&& tar xvf zlib-1.2.12.tar.gz \
-&& cd zlib-1.2.12 \
+RUN wget -q https://zlib.net/zlib-1.2.13.tar.gz \
+&& tar xvf zlib-1.2.13.tar.gz \
+&& cd zlib-1.2.13 \
 && ./configure --prefix=$QEMU_LD_PREFIX \
 && make \
-&& make install && cd .. && rm zlib-1.2.12.tar.gz && rm -rf zlib-1.2.12
+&& make install && cd .. && rm zlib-1.2.13.tar.gz && rm -rf zlib-1.2.13
 
 RUN apt-get install -y --no-install-recommends unzip tclsh \
 && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
@@ -87,13 +112,28 @@ RUN git clone --recursive /tmp/lightning . && \
 
 ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
-FROM arm32v7/debian:buster-slim as final
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+&& pip3 install -U pip \
+&& pip3 install -U wheel \
+&& /root/.local/bin/poetry install
+
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
+make DEVELOPER=${DEVELOPER} && \
+/root/.local/bin/poetry run make install
+
+FROM arm32v7/debian:bullseye-slim as final
 COPY --from=downloader /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static
 COPY --from=downloader /opt/tini /usr/bin/tini
-RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
-    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      socat \
+      inotify-tools \
+      python3 \
+      python3-pip \
+      libpq5 && \
+    rm -rf /var/lib/apt/lists/*
 
 ENV LIGHTNINGD_DATA=/root/.lightning
 ENV LIGHTNINGD_RPC_PORT=9835

contrib/docker/linuxarm64v8.Dockerfile

@@ -5,7 +5,7 @@
 # * final: Copy the binaries required at runtime
 # The resulting image uploaded to dockerhub will only contain what is needed for runtime.
 # From the root of the repository, run "docker build -t yourimage:yourtag -f contrib/linuxarm64v8.Dockerfile ."
-FROM debian:buster-slim as downloader
+FROM debian:bullseye-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -18,7 +18,7 @@ RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.
     && echo "7c5463f55393985ee22357d976758aaaecd08defb3c5294d353732018169b019 /opt/tini" | sha256sum -c - \
     && chmod +x /opt/tini
 
-		ENV GROESTLCOIN_VERSION 2.20.1
+		ENV GROESTLCOIN_VERSION 22.0
 		ENV GROESTLCOIN_TARBALL groestlcoin-${GROESTLCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 		ENV GROESTLCOIN_URL https://github.com/Groestlcoin/groestlcoin/releases/download/v$GROESTLCOIN_VERSION/$GROESTLCOIN_TARBALL
 		ENV GROESTLCOIN_ASC_URL https://github.com/Groestlcoin/groestlcoin/releases/download/v$GROESTLCOIN_VERSION/SHA256SUMS.asc
@@ -40,11 +40,36 @@ RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.
 		    && tar -xzvf $GROESTLCOIN_TARBALL $BD/groestlcoin-cli --strip-components=1 \
 		    && rm $GROESTLCOIN_TARBALL
 
-FROM debian:buster-slim as builder
+FROM debian:bullseye-slim as builder
 
 ENV LIGHTNINGD_VERSION=master
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates autoconf automake build-essential gettext git libtool python3 python3-pip python3-setuptools python3-mako wget gnupg dirmngr git lowdown \
-  libc6-arm64-cross gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        autoconf \
+        automake \
+        build-essential \
+        ca-certificates \
+        curl \
+        dirmngr \
+        gettext \
+        git \
+        gnupg \
+        libpq-dev \
+        libtool \
+        libffi-dev \
+        python3 \
+        python3-dev \
+        python3-mako \
+        python3-pip \
+        python3-venv \
+        python3-setuptools \
+        wget && \
+    # arm64v8 compilers
+    apt-get install -qq -y --no-install-recommends \
+        libc6-arm64-cross \
+        gcc-aarch64-linux-gnu \
+        g++-aarch64-linux-gnu
 
 ENV target_host=aarch64-linux-gnu
 
@@ -57,12 +82,12 @@ STRIP=${target_host}-strip \
 QEMU_LD_PREFIX=/usr/${target_host} \
 HOST=${target_host}
 
-RUN wget -q https://zlib.net/zlib-1.2.12.tar.gz \
-&& tar xvf zlib-1.2.12.tar.gz \
-&& cd zlib-1.2.12 \
+RUN wget -q https://zlib.net/zlib-1.2.13.tar.gz \
+&& tar xvf zlib-1.2.13.tar.gz \
+&& cd zlib-1.2.13 \
 && ./configure --prefix=$QEMU_LD_PREFIX \
 && make \
-&& make install && cd .. && rm zlib-1.2.12.tar.gz && rm -rf zlib-1.2.12
+&& make install && cd .. && rm zlib-1.2.13.tar.gz && rm -rf zlib-1.2.13
 
 RUN apt-get install -y --no-install-recommends unzip tclsh \
 && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
@@ -86,13 +111,28 @@ RUN git clone --recursive /tmp/lightning . && \
 
 ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
-FROM arm64v8/debian:buster-slim as final
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+&& pip3 install -U pip \
+&& pip3 install -U wheel \
+&& /root/.local/bin/poetry install
+
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
+make DEVELOPER=${DEVELOPER} && \
+/root/.local/bin/poetry run make install
+
+FROM arm64v8/debian:bullseye-slim as final
 COPY --from=downloader /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
 COPY --from=downloader /opt/tini /usr/bin/tini
-RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
-    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      socat \
+      inotify-tools \
+      python3 \
+      python3-pip \
+      libpq5 && \
+    rm -rf /var/lib/apt/lists/*
 
 ENV LIGHTNINGD_DATA=/root/.lightning
 ENV LIGHTNINGD_RPC_PORT=9835