From 14f3da89433af5e6bd9810de367112fe388a8fa8 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 7 Nov 2024 16:20:13 +0100 Subject: [PATCH] testserver: First iteration of the mock Greenlight server This uses the `gl-testing` library, and builds a standalone server to test against. We currently expose four interfaces: - The scheduler interface as the main entrypoint to the service - The GRPC-Web proxy to develop browser apps and extensions against Greenlight. - The `bitcoind` interface, so you can generate blocks and confirm transactions without lengthy wait times - The node's grpc interface directly to work against a single user's node All of these will listen to random ports initially. We write a small file `metadata.json` which contains the URIs and ports for the first three, while the node's URI can be retrieved from the scheduler, since these are spawned on demand as users register. --- libs/gl-testserver/README.md | 0 libs/gl-testserver/gltestserver/__init__.py | 0 libs/gl-testserver/gltestserver/__main__.py | 140 ++++++++++++++++++++ libs/gl-testserver/pyproject.toml | 17 +++ libs/gl-testserver/tests/test_server.py | 107 +++++++++++++++ pyproject.toml | 2 +- 6 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 libs/gl-testserver/README.md create mode 100644 libs/gl-testserver/gltestserver/__init__.py create mode 100644 libs/gl-testserver/gltestserver/__main__.py create mode 100644 libs/gl-testserver/pyproject.toml create mode 100644 libs/gl-testserver/tests/test_server.py diff --git a/libs/gl-testserver/README.md b/libs/gl-testserver/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/libs/gl-testserver/gltestserver/__init__.py b/libs/gl-testserver/gltestserver/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libs/gl-testserver/gltestserver/__main__.py b/libs/gl-testserver/gltestserver/__main__.py new file mode 100644 index 000000000..3e0b658ff --- /dev/null +++ b/libs/gl-testserver/gltestserver/__main__.py @@ -0,0 +1,140 @@ +import json +from dataclasses import dataclass + +import time + +from rich.console import Console +from rich.pretty import pprint +from rich import inspect +from pathlib import Path +from gltesting import fixtures +import gltesting +from inspect import isgeneratorfunction +import click +import logging +from rich.logging import RichHandler +from pyln.testing.utils import BitcoinD +from typing import Any, List + + +console = Console() +logging.basicConfig( + level="DEBUG", + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler(rich_tracebacks=True, tracebacks_suppress=[click], console=console) + ], +) +logger = logging.getLogger("gltestserver") + + +@dataclass +class TestServer: + directory: Path + bitcoind: BitcoinD + scheduler: gltesting.scheduler.Scheduler + finalizers: List[Any] + clients: gltesting.clients.Clients + grpc_web_proxy: gltesting.grpcweb.GrpcWebProxy + + def stop(self): + for f in self.finalizers[::-1]: + try: + f() + except StopIteration: + continue + except Exception as e: + logger.warn(f"Unexpected exception tearing down server: {e}") + + def metadata(self): + """Construct a dict of config values for this TestServer.""" + return { + "scheduler_grpc_uri": self.scheduler.grpc_addr, + "grpc_web_proxy_uri": f"http://localhost:{self.grpc_web_proxy.web_port}", + "bitcoind_rpc_uril": f"http://rpcuser:rpcpass@localhost:{self.bitcoind.rpcport}", + } + + +def build(): + # List of teardown functions to call in reverse order. + finalizers = [] + + def callfixture(f, *args, **kwargs): + """Small shim to bypass the pytest decorator.""" + F = f.__pytest_wrapped__.obj + + if isgeneratorfunction(F): + it = F(*args, **kwargs) + v = it.__next__() + finalizers.append(it.__next__) + return v + else: + return F(*args, **kwargs) + + directory = Path("/tmp/gl-testserver") + + cert_directory = callfixture(fixtures.cert_directory, directory) + root_id = callfixture(fixtures.root_id, cert_directory) + users_id = callfixture(fixtures.users_id) + nobody_id = callfixture(fixtures.nobody_id, cert_directory) + scheduler_id = callfixture(fixtures.scheduler_id, cert_directory) + paths = callfixture(fixtures.paths) + bitcoind = callfixture( + fixtures.bitcoind, + directory=directory, + teardown_checks=None, + ) + scheduler = callfixture( + fixtures.scheduler, scheduler_id=scheduler_id, bitcoind=bitcoind + ) + + clients = callfixture( + fixtures.clients, directory=directory, scheduler=scheduler, nobody_id=nobody_id + ) + + node_grpc_web_server = callfixture( + fixtures.node_grpc_web_proxy, scheduler=scheduler + ) + + return TestServer( + directory=directory, + bitcoind=bitcoind, + finalizers=finalizers, + scheduler=scheduler, + clients=clients, + grpc_web_proxy=node_grpc_web_server, + ) + + +@click.group() +def cli(): + pass + + +@cli.command() +def run(): + gl = build() + try: + meta = gl.metadata() + metafile = gl.directory / "metadata.json" + logger.debug(f"Writing testserver metadata to {metafile}") + with metafile.open(mode="w") as f: + json.dump(meta, f) + + pprint(meta) + logger.info( + f"Server is up and running with the above config values. To stop press Ctrl-C." + ) + time.sleep(1800) + except Exception as e: + logger.warning(f"Caught exception running testserver: {e}") + pass + finally: + logger.info("Stopping gl-testserver") + # Now tear things down again. + gl.stop() + + +if __name__ == "__main__": + cli() diff --git a/libs/gl-testserver/pyproject.toml b/libs/gl-testserver/pyproject.toml new file mode 100644 index 000000000..e6eac8497 --- /dev/null +++ b/libs/gl-testserver/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "gltestserver" +version = "0.1.0" +description = "A standalone test server implementing the public Greenlight interfaces" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "click>=8.1.7", + "gltesting", + "rich>=13.9.3", +] + +[project.scripts] +gltestserver = 'gltestserver.__main__:cli' + +[tool.uv.sources] +gltesting = { workspace = true } diff --git a/libs/gl-testserver/tests/test_server.py b/libs/gl-testserver/tests/test_server.py new file mode 100644 index 000000000..e21afde0f --- /dev/null +++ b/libs/gl-testserver/tests/test_server.py @@ -0,0 +1,107 @@ +# We do not import `gl-testing` or `pyln-testing` since the +# `gl-testserver` is intended to run tests externally from a python +# environment. We will use `gl-client-py` to interact with it though. +# Ok, one exception, `TailableProc` is used to run and tail the +# `gl-testserver`. + +import shutil +import tempfile +import os +import pytest +from pyln.testing.utils import TailableProc +import json +import signal +from pathlib import Path + + +@pytest.fixture +def test_name(request): + yield request.function.__name__ + + +@pytest.fixture(scope="session") +def test_base_dir(): + d = os.getenv("TEST_DIR", "/tmp") + directory = tempfile.mkdtemp(prefix="ltests-", dir=d) + print("Running tests in {}".format(directory)) + + yield directory + + +@pytest.fixture +def directory(request, test_base_dir, test_name): + """Return a per-test specific directory. + + This makes a unique test-directory even if a test is rerun multiple times. + + """ + directory = os.path.join(test_base_dir, test_name) + request.node.has_errors = False + + if not os.path.exists(directory): + os.makedirs(directory) + + yield directory + + # This uses the status set in conftest.pytest_runtest_makereport to + # determine whether we succeeded or failed. Outcome can be None if the + # failure occurs during the setup phase, hence the use to getattr instead + # of accessing it directly. + rep_call = getattr(request.node, "rep_call", None) + outcome = "passed" if rep_call is None else rep_call.outcome + failed = not outcome or request.node.has_errors or outcome != "passed" + + if not failed: + try: + shutil.rmtree(directory) + except OSError: + # Usually, this means that e.g. valgrind is still running. Wait + # a little and retry. + files = [ + os.path.join(dp, f) for dp, dn, fn in os.walk(directory) for f in fn + ] + print("Directory still contains files: ", files) + print("... sleeping then retrying") + time.sleep(10) + shutil.rmtree(directory) + else: + logging.debug( + "Test execution failed, leaving the test directory {} intact.".format( + directory + ) + ) + + +class TestServer(TailableProc): + def __init__(self, directory): + TailableProc.__init__(self, outputDir=directory) + self.cmd_line = [ + "python3", + str(Path(__file__).parent / ".." / "gltestserver" / "__main__.py"), + "run", + ] + + def start(self): + TailableProc.start(self) + self.wait_for_log(r"Ctrl-C") + + def stop(self): + self.proc.send_signal(signal.SIGTERM) + self.proc.wait() + + +@pytest.fixture +def testserver(directory): + ts = TestServer(directory=directory) + ts.start() + + + metadata = json.load(open(f'{directory}/metadata.json')) + pprint(metadata) + + yield ts + ts.stop() + + +def test_start(testserver): + print(TailableProc) diff --git a/pyproject.toml b/pyproject.toml index 76029b65b..410b4315f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,4 +40,4 @@ pillow = "^9.5.0" python-lsp-server = "^1.10.0" [tool.uv.workspace] -members = ["libs/gl-testing"] +members = ["libs/gl-testing", "libs/gl-testserver"]