From 888f42d9cee6514dccfd0762150733b08268426f Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 7 Nov 2024 16:19:01 +0100 Subject: [PATCH 1/4] testing: Do not error out if the certs already exists --- libs/gl-testing/gltesting/certs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/gl-testing/gltesting/certs.py b/libs/gl-testing/gltesting/certs.py index e191bc05e..dbbc80138 100644 --- a/libs/gl-testing/gltesting/certs.py +++ b/libs/gl-testing/gltesting/certs.py @@ -210,7 +210,7 @@ def gencert(idpath): for f in path: if os.path.exists(f): logging.info(f"Not overwriting existing file {f}") - return + return Identity.from_path(idpath) tmpcsr = tempfile.NamedTemporaryFile(mode="w") json.dump(mycsr, tmpcsr) From bfa7a9a7e923561bcdade0da3b5a0ea8e0b740fd Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 7 Nov 2024 16:20:13 +0100 Subject: [PATCH 2/4] testserver: First iteration of the mock Greenlight server This uses the `gl-testing` library, and builds a standalone server to test against. We currently expose four interfaces: - The scheduler interface as the main entrypoint to the service - The GRPC-Web proxy to develop browser apps and extensions against Greenlight. - The `bitcoind` interface, so you can generate blocks and confirm transactions without lengthy wait times - The node's grpc interface directly to work against a single user's node All of these will listen to random ports initially. We write a small file `metadata.json` which contains the URIs and ports for the first three, while the node's URI can be retrieved from the scheduler, since these are spawned on demand as users register. --- libs/gl-testserver/README.md | 0 libs/gl-testserver/gltestserver/__init__.py | 0 libs/gl-testserver/gltestserver/__main__.py | 140 ++++++++++++++++++++ libs/gl-testserver/pyproject.toml | 17 +++ libs/gl-testserver/tests/test_server.py | 107 +++++++++++++++ pyproject.toml | 2 +- 6 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 libs/gl-testserver/README.md create mode 100644 libs/gl-testserver/gltestserver/__init__.py create mode 100644 libs/gl-testserver/gltestserver/__main__.py create mode 100644 libs/gl-testserver/pyproject.toml create mode 100644 libs/gl-testserver/tests/test_server.py diff --git a/libs/gl-testserver/README.md b/libs/gl-testserver/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/libs/gl-testserver/gltestserver/__init__.py b/libs/gl-testserver/gltestserver/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libs/gl-testserver/gltestserver/__main__.py b/libs/gl-testserver/gltestserver/__main__.py new file mode 100644 index 000000000..693e2b1de --- /dev/null +++ b/libs/gl-testserver/gltestserver/__main__.py @@ -0,0 +1,140 @@ +import json +from dataclasses import dataclass + +import time + +from rich.console import Console +from rich.pretty import pprint +from rich import inspect +from pathlib import Path +from gltesting import fixtures +import gltesting +from inspect import isgeneratorfunction +import click +import logging +from rich.logging import RichHandler +from pyln.testing.utils import BitcoinD +from typing import Any, List + + +console = Console() +logging.basicConfig( + level="DEBUG", + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler(rich_tracebacks=True, tracebacks_suppress=[click], console=console) + ], +) +logger = logging.getLogger("gltestserver") + + +@dataclass +class TestServer: + directory: Path + bitcoind: BitcoinD + scheduler: gltesting.scheduler.Scheduler + finalizers: List[Any] + clients: gltesting.clients.Clients + grpc_web_proxy: gltesting.grpcweb.GrpcWebProxy + + def stop(self): + for f in self.finalizers[::-1]: + try: + f() + except StopIteration: + continue + except Exception as e: + logger.warn(f"Unexpected exception tearing down server: {e}") + + def metadata(self): + """Construct a dict of config values for this TestServer.""" + return { + "scheduler_grpc_uri": self.scheduler.grpc_addr, + "grpc_web_proxy_uri": f"http://localhost:{self.grpc_web_proxy.web_port}", + "bitcoind_rpc_uri": f"http://rpcuser:rpcpass@localhost:{self.bitcoind.rpcport}", + } + + +def build(): + # List of teardown functions to call in reverse order. + finalizers = [] + + def callfixture(f, *args, **kwargs): + """Small shim to bypass the pytest decorator.""" + F = f.__pytest_wrapped__.obj + + if isgeneratorfunction(F): + it = F(*args, **kwargs) + v = it.__next__() + finalizers.append(it.__next__) + return v + else: + return F(*args, **kwargs) + + directory = Path("/tmp/gl-testserver") + + cert_directory = callfixture(fixtures.cert_directory, directory) + root_id = callfixture(fixtures.root_id, cert_directory) + users_id = callfixture(fixtures.users_id) + nobody_id = callfixture(fixtures.nobody_id, cert_directory) + scheduler_id = callfixture(fixtures.scheduler_id, cert_directory) + paths = callfixture(fixtures.paths) + bitcoind = callfixture( + fixtures.bitcoind, + directory=directory, + teardown_checks=None, + ) + scheduler = callfixture( + fixtures.scheduler, scheduler_id=scheduler_id, bitcoind=bitcoind + ) + + clients = callfixture( + fixtures.clients, directory=directory, scheduler=scheduler, nobody_id=nobody_id + ) + + node_grpc_web_server = callfixture( + fixtures.node_grpc_web_proxy, scheduler=scheduler + ) + + return TestServer( + directory=directory, + bitcoind=bitcoind, + finalizers=finalizers, + scheduler=scheduler, + clients=clients, + grpc_web_proxy=node_grpc_web_server, + ) + + +@click.group() +def cli(): + pass + + +@cli.command() +def run(): + gl = build() + try: + meta = gl.metadata() + metafile = gl.directory / "metadata.json" + logger.debug(f"Writing testserver metadata to {metafile}") + with metafile.open(mode="w") as f: + json.dump(meta, f) + + pprint(meta) + logger.info( + f"Server is up and running with the above config values. To stop press Ctrl-C." + ) + time.sleep(1800) + except Exception as e: + logger.warning(f"Caught exception running testserver: {e}") + pass + finally: + logger.info("Stopping gl-testserver") + # Now tear things down again. + gl.stop() + + +if __name__ == "__main__": + cli() diff --git a/libs/gl-testserver/pyproject.toml b/libs/gl-testserver/pyproject.toml new file mode 100644 index 000000000..e6eac8497 --- /dev/null +++ b/libs/gl-testserver/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "gltestserver" +version = "0.1.0" +description = "A standalone test server implementing the public Greenlight interfaces" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "click>=8.1.7", + "gltesting", + "rich>=13.9.3", +] + +[project.scripts] +gltestserver = 'gltestserver.__main__:cli' + +[tool.uv.sources] +gltesting = { workspace = true } diff --git a/libs/gl-testserver/tests/test_server.py b/libs/gl-testserver/tests/test_server.py new file mode 100644 index 000000000..e21afde0f --- /dev/null +++ b/libs/gl-testserver/tests/test_server.py @@ -0,0 +1,107 @@ +# We do not import `gl-testing` or `pyln-testing` since the +# `gl-testserver` is intended to run tests externally from a python +# environment. We will use `gl-client-py` to interact with it though. +# Ok, one exception, `TailableProc` is used to run and tail the +# `gl-testserver`. + +import shutil +import tempfile +import os +import pytest +from pyln.testing.utils import TailableProc +import json +import signal +from pathlib import Path + + +@pytest.fixture +def test_name(request): + yield request.function.__name__ + + +@pytest.fixture(scope="session") +def test_base_dir(): + d = os.getenv("TEST_DIR", "/tmp") + directory = tempfile.mkdtemp(prefix="ltests-", dir=d) + print("Running tests in {}".format(directory)) + + yield directory + + +@pytest.fixture +def directory(request, test_base_dir, test_name): + """Return a per-test specific directory. + + This makes a unique test-directory even if a test is rerun multiple times. + + """ + directory = os.path.join(test_base_dir, test_name) + request.node.has_errors = False + + if not os.path.exists(directory): + os.makedirs(directory) + + yield directory + + # This uses the status set in conftest.pytest_runtest_makereport to + # determine whether we succeeded or failed. Outcome can be None if the + # failure occurs during the setup phase, hence the use to getattr instead + # of accessing it directly. + rep_call = getattr(request.node, "rep_call", None) + outcome = "passed" if rep_call is None else rep_call.outcome + failed = not outcome or request.node.has_errors or outcome != "passed" + + if not failed: + try: + shutil.rmtree(directory) + except OSError: + # Usually, this means that e.g. valgrind is still running. Wait + # a little and retry. + files = [ + os.path.join(dp, f) for dp, dn, fn in os.walk(directory) for f in fn + ] + print("Directory still contains files: ", files) + print("... sleeping then retrying") + time.sleep(10) + shutil.rmtree(directory) + else: + logging.debug( + "Test execution failed, leaving the test directory {} intact.".format( + directory + ) + ) + + +class TestServer(TailableProc): + def __init__(self, directory): + TailableProc.__init__(self, outputDir=directory) + self.cmd_line = [ + "python3", + str(Path(__file__).parent / ".." / "gltestserver" / "__main__.py"), + "run", + ] + + def start(self): + TailableProc.start(self) + self.wait_for_log(r"Ctrl-C") + + def stop(self): + self.proc.send_signal(signal.SIGTERM) + self.proc.wait() + + +@pytest.fixture +def testserver(directory): + ts = TestServer(directory=directory) + ts.start() + + + metadata = json.load(open(f'{directory}/metadata.json')) + pprint(metadata) + + yield ts + ts.stop() + + +def test_start(testserver): + print(TailableProc) diff --git a/pyproject.toml b/pyproject.toml index 76029b65b..410b4315f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,4 +40,4 @@ pillow = "^9.5.0" python-lsp-server = "^1.10.0" [tool.uv.workspace] -members = ["libs/gl-testing"] +members = ["libs/gl-testing", "libs/gl-testserver"] From 469e7c28315bfa84e7f8fc16b2dec2a48b274381 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 7 Nov 2024 17:02:18 +0100 Subject: [PATCH 3/4] gltestserver: Make the top-level directory configurable The TLD is the root for the tree of resources we are going to spawn, so making this configurable allows us to run arbitrarily many instances on a single developer machine. This is useful if we'd like to run many tests in parallel. --- libs/gl-testserver/gltestserver/__main__.py | 54 +++++++++++++-------- libs/gl-testserver/tests/test_server.py | 26 ++++++---- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/libs/gl-testserver/gltestserver/__main__.py b/libs/gl-testserver/gltestserver/__main__.py index 693e2b1de..5c7af870e 100644 --- a/libs/gl-testserver/gltestserver/__main__.py +++ b/libs/gl-testserver/gltestserver/__main__.py @@ -1,20 +1,18 @@ -import json from dataclasses import dataclass - -import time - -from rich.console import Console -from rich.pretty import pprint -from rich import inspect -from pathlib import Path from gltesting import fixtures -import gltesting from inspect import isgeneratorfunction -import click -import logging -from rich.logging import RichHandler +from pathlib import Path from pyln.testing.utils import BitcoinD +from rich.console import Console +from rich.logging import RichHandler +from rich.pretty import pprint from typing import Any, List +import click +import gltesting +import json +import logging +import tempfile +import time console = Console() @@ -56,7 +54,7 @@ def metadata(self): } -def build(): +def build(base_dir: Path): # List of teardown functions to call in reverse order. finalizers = [] @@ -72,14 +70,14 @@ def callfixture(f, *args, **kwargs): else: return F(*args, **kwargs) - directory = Path("/tmp/gl-testserver") + directory = base_dir / "gl-testserver" cert_directory = callfixture(fixtures.cert_directory, directory) - root_id = callfixture(fixtures.root_id, cert_directory) - users_id = callfixture(fixtures.users_id) + _root_id = callfixture(fixtures.root_id, cert_directory) + _users_id = callfixture(fixtures.users_id) nobody_id = callfixture(fixtures.nobody_id, cert_directory) scheduler_id = callfixture(fixtures.scheduler_id, cert_directory) - paths = callfixture(fixtures.paths) + _paths = callfixture(fixtures.paths) bitcoind = callfixture( fixtures.bitcoind, directory=directory, @@ -113,18 +111,34 @@ def cli(): @cli.command() -def run(): - gl = build() +@click.option( + "--directory", + type=click.Path(), + help=""" + Set the top-level directory for the testserver. This can be used to run + multiple instances isolated from each other, by giving each isntance a + different top-level directory. Defaults to '/tmp/' + """, +) +def run(directory): + """Start a gl-testserver instance to test against.""" + if not directory: + directory = Path(tempfile.gettempdir()) + else: + directory = Path(directory) + + gl = build(base_dir=directory) try: meta = gl.metadata() metafile = gl.directory / "metadata.json" + metafile.parent.mkdir(parents=True, exist_ok=True) logger.debug(f"Writing testserver metadata to {metafile}") with metafile.open(mode="w") as f: json.dump(meta, f) pprint(meta) logger.info( - f"Server is up and running with the above config values. To stop press Ctrl-C." + "Server is up and running with the above config values. To stop press Ctrl-C." ) time.sleep(1800) except Exception as e: diff --git a/libs/gl-testserver/tests/test_server.py b/libs/gl-testserver/tests/test_server.py index e21afde0f..4e9790950 100644 --- a/libs/gl-testserver/tests/test_server.py +++ b/libs/gl-testserver/tests/test_server.py @@ -4,14 +4,16 @@ # Ok, one exception, `TailableProc` is used to run and tail the # `gl-testserver`. -import shutil -import tempfile -import os -import pytest +from pathlib import Path from pyln.testing.utils import TailableProc import json +import logging +import os +import pytest +import shutil import signal -from pathlib import Path +import tempfile +import time @pytest.fixture @@ -79,7 +81,9 @@ def __init__(self, directory): "python3", str(Path(__file__).parent / ".." / "gltestserver" / "__main__.py"), "run", + f"--directory={directory}", ] + self.directory = Path(directory) def start(self): TailableProc.start(self) @@ -89,19 +93,21 @@ def stop(self): self.proc.send_signal(signal.SIGTERM) self.proc.wait() + def metadata(self): + metadata = json.load( + (self.directory / "gl-testserver" / "metadata.json").open(mode="r") + ) + return metadata + @pytest.fixture def testserver(directory): ts = TestServer(directory=directory) ts.start() - - metadata = json.load(open(f'{directory}/metadata.json')) - pprint(metadata) - yield ts ts.stop() def test_start(testserver): - print(TailableProc) + print(testserver.metadata()) From d843a52ef7424e6cd9db39fad89cef1d457797c7 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 7 Nov 2024 17:48:57 +0100 Subject: [PATCH 4/4] docs: Add reference/testserver.md to document the `gl-testserver` --- docs/mkdocs.yml | 1 + docs/src/reference/testserver.md | 111 +++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 docs/src/reference/testserver.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index e88a87c7b..e2440d2e6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -83,6 +83,7 @@ nav: - Client Libraries: reference/client-libraries.md - Credentials: reference/creds.md - reference/node-domain.md + - Testserver: reference/testserver.md - Certificates: reference/certs.md - Security: reference/security.md - LSP Integration: reference/lsp.md diff --git a/docs/src/reference/testserver.md b/docs/src/reference/testserver.md new file mode 100644 index 000000000..e0e149e4e --- /dev/null +++ b/docs/src/reference/testserver.md @@ -0,0 +1,111 @@ +# The `gl-testserver` + +The `gl-testserver` is a standalone version, of the `gl-testing` +framework, allowing you to test against a mock Greenlight server, +independently of your programming language and development +environment. + +The goal of the `gl-testing` package is to enable developers to test +locally against a mock Greenlight server. This has a number of +advantages: + + - **Speed**: by not involving the network, you can test without + latency slowing you down. The tests also run on a `regtest` + network, allowing you to generate blocks and confirm transactions + without the need to wait. + - **Cost**: the `prod` network is not free, and since tests may + consume arbitrary resources, which are then not cleaned up (see + next point), repeatedly running them could incur costs. We see this + as a bad incentive to minimize testing during development, and + `gl-testing` allows you to use only local resources that can then + also be reclaimed, making testing free, and hopefully encouraging + to test more. + - **Reproducibility**: The `prod` network does not allow cleaning up + test resources, since there might be an actual user using + them. This means that test artifacts persist between runs, + resulting in a non-reproducible environment for + testing. `gl-testing` running locally allows cleaning up resources, + thus enabling reproducible tests. + +However, the downside of `gl-testing` is that is coupled with `python` +as programming language and `pytest` as test runner. This is where +`gl-testserver` comes in: by bundling all the fixtures and startup +logic into a standalone binary we can pull up an instance in a matter +of seconds, test and develop against it, and then tear it down at the +end of our session. + +## How to use `gl-testserver` + +It's probably easiest to use `uv` to run the script from the source +tree. Please see the [`uv` installation instructions][uv/install] for +how to get started installing `uv` then come back here. + +Executing `uv run gltestserver` is the entrypoint for the tool: + +```bash +gltestserver +Usage: gltestserver [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + run Start a gl-testserver instance to test against. +``` + +Currently there is only the `run` subcommand which will start the +testserver, binding the scheduler GRPC interface, the `bitcoind` RPC +interface, and the GRPC-web proxy to random ports on `localhost`. + + +```bash +gltestserver run --help +Usage: gltestserver run [OPTIONS] + + Start a gl-testserver instance to test against. + +Options: + --directory PATH Set the top-level directory for the testserver. This can + be used to run multiple instances isolated from each + other, by giving each isntance a different top-level + directory. Defaults to '/tmp/' + --help Show this message and exit +``` + +In order to identify the ports for the current instance you can either +see the end of the output of the command, which contains +pretty-printed key-value pairs, or load the `metadata.json` file +containing the port references to use from the `gl-testserver` +sub-directory (`/tmp/gl-testserver` if you haven't specified the +`--directory` option). + +!!! note "Running multiple tests in parallel" + As the help text above already points out it is possible to run as many + instances of the testserver concurrently as you want, by specifying + separate `--directory` options in each call. + + This is particularly useful if you want to run multiple tests in parallel + to speed up the test runs. It is also the core reason why ports are + randomized rather than using fixed ports per interface, as concurrent + instances would conflict, and the isolation between tests could be + compromised. + +Once started you will see the following lines printed: + +``` +Writing testserver metadata to /tmp/gl-testserver/metadata.json +{ + 'scheduler_grpc_uri': 'https://localhost:38209', + 'grpc_web_proxy_uri': 'http://localhost:35911', + 'bitcoind_rpc_uri': 'http://rpcuser:rpcpass@localhost:44135' +} +Server is up and running with the above config values. To stop press Ctrl-C. +``` + +At this point you can use the URIs that are printed to interact with +the services, or use `Ctrl-C` to stop the server. When running in a +test environment you can send `SIGTERM` to the process and it'll also +shut down gracefully, cleaning up the processes, but leaving the data +created during the test in the directory. + +[uv/install]: https://docs.astral.sh/uv/getting-started/installation/