From 50d46745b7e86b632fa3d6f5b23a2b4d60085153 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 13:57:08 +0100 Subject: [PATCH 01/86] adding memory --- src/osparc_control/memory.py | 90 +++++++++++++++++++++++ src/osparc_control/types.py | 3 + tests/test_memory.py | 134 +++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 src/osparc_control/memory.py create mode 100644 src/osparc_control/types.py create mode 100644 tests/test_memory.py diff --git a/src/osparc_control/memory.py b/src/osparc_control/memory.py new file mode 100644 index 0000000..4768e03 --- /dev/null +++ b/src/osparc_control/memory.py @@ -0,0 +1,90 @@ +from typing import Dict, Union, List, Tuple +from collections import OrderedDict +from .types import AcceptedValues +from .errors import ( + CollectionIsFullException, + KeyIsAlreadyPresent, + KeyWasNotFoundException, + TimeIndexMissingException, + TimeIndexTooSmallOrNotExistingException, +) + + +class MemoryStore: + """Size limited time indexed memory store""" + + def __init__(self) -> None: + self._store: Dict[str, OrderedDict[float, AcceptedValues]] = {} + self._max_sizes: Dict[str, int] = {} + self._last_inserted_time_index: Dict[str, float] = {} + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self._store} {self._max_sizes}>" + + def _ensure_key(self, key: str) -> None: + if key not in self._store: + raise KeyWasNotFoundException(f"Key {key} was not found") + + def init_collection(self, key: str, max_size: int) -> None: + if key in self._store: + raise KeyIsAlreadyPresent(key, self._store[key]) + + self._store[key] = OrderedDict() + self._max_sizes[key] = max_size + + def set_value(self, key: str, time_index: float, value: AcceptedValues) -> None: + self._ensure_key(key) + + # ensure new previous times are not inserter + # it is ok to change their values but not to add new ones + last_inserted_time_index = self._last_inserted_time_index.get( + key, -float("inf") + ) + if last_inserted_time_index > time_index and time_index not in self._store[key]: + raise TimeIndexTooSmallOrNotExistingException( + time_index, last_inserted_time_index + ) + + self._store[key][time_index] = value + self._last_inserted_time_index[key] = time_index + + # collection is size limited, avoid memory blow up, user is required + # to specify the size before using it + if len(self._store[key]) > self._max_sizes[key]: + raise CollectionIsFullException(key=key, max_items=self._max_sizes[key]) + + def size_of(self, key: str) -> int: + self._ensure_key(key) + return len(self._store[key]) + + def get_value( + self, key: str, time_index: float, default: Union[AcceptedValues, None] = None + ) -> Union[AcceptedValues, None]: + # maybe this should raise if time_index is missing? + self._ensure_key(key) + + return self._store[key].get(time_index, default) + + def get_interval( + self, key: str, time_index_start: float, time_index_stop: float + ) -> List[Tuple[float, AcceptedValues]]: + """works exactly like list slicing""" + self._ensure_key(key) + + collection: OrderedDict[float, AcceptedValues] = self._store[key] + + if time_index_start not in collection: + raise TimeIndexMissingException(time_index_start) + + if time_index_stop not in collection: + raise TimeIndexMissingException(time_index_stop) + + # map key_index to order than you search for them + key_to_index = {key: i for i, key in enumerate(collection.keys())} + + start_index = key_to_index[time_index_start] + stop_index = key_to_index[time_index_stop] + + collection_items = list(collection.items()) + + return collection_items[start_index:stop_index] diff --git a/src/osparc_control/types.py b/src/osparc_control/types.py new file mode 100644 index 0000000..6d323c3 --- /dev/null +++ b/src/osparc_control/types.py @@ -0,0 +1,3 @@ +from typing import Union + +AcceptedValues = Union[str, int, float, bytes] diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..bef034c --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,134 @@ +import pytest + +from osparc_control.memory import MemoryStore +from osparc_control.types import AcceptedValues +from osparc_control.errors import ( + CollectionIsFullException, + KeyIsAlreadyPresent, + KeyWasNotFoundException, + TimeIndexMissingException, + TimeIndexTooSmallOrNotExistingException, +) + + +@pytest.fixture +def key() -> str: + return "test_key" + + +@pytest.fixture +def max_size() -> int: + return 10 + + +@pytest.fixture +def memory_store() -> MemoryStore: + return MemoryStore() + + +@pytest.fixture +def fille_memory_store( + key: str, memory_store: MemoryStore, max_size: int +) -> MemoryStore: + memory_store.init_collection(key, max_size) + + for t in range(max_size): + memory_store.set_value(key, t, f"v{t}") + + return memory_store + + +def test_memory_store_init_set_get(key: str, memory_store: MemoryStore, max_size: int): + memory_store.init_collection(key, max_size) + + for t in range(max_size): + memory_store.set_value(key, t, f"v{t}") + + print(memory_store) + + assert memory_store.get_value(key, 0) == "v0" + assert memory_store.get_value(key, 9) == "v9" + + +def test_key_is_not_initialized(key: str, memory_store: MemoryStore): + with pytest.raises(KeyWasNotFoundException): + memory_store.get_value(key, 0) + + +def test_key_is_not_present(key: str, memory_store: MemoryStore): + with pytest.raises(KeyWasNotFoundException): + memory_store.set_value(key, 0, "10") + + +def test_list_is_full(key: str, memory_store: MemoryStore, max_size: int): + memory_store.init_collection(key, max_size) + + for t in range(max_size): + memory_store.set_value(key, t, f"v{t}") + + assert memory_store.size_of(key) == max_size + + with pytest.raises(CollectionIsFullException): + memory_store.set_value(key, 10, f"v10") + + +def test_initialize_the_same_key_twice_fails( + key: str, memory_store: MemoryStore, max_size: int +): + memory_store.init_collection(key, max_size) + assert key in memory_store._store + assert key in memory_store._max_sizes + + with pytest.raises(KeyIsAlreadyPresent): + memory_store.init_collection(key, max_size) + + +def test_base_values(key: str, memory_store: MemoryStore, max_size: int): + memory_store.init_collection(key, max_size) + + def _assert_get_is_set(index: float, value: AcceptedValues) -> None: + memory_store.set_value(key, index, value) + stored = memory_store.get_value(key, index) + assert stored == value + + _assert_get_is_set(0, 1) + _assert_get_is_set(0, 1.0) + _assert_get_is_set(0, "1") + _assert_get_is_set(0, b"1") + + +def test_only_allow_increasing_times( + key: str, memory_store: MemoryStore, max_size: int +): + memory_store.init_collection(key, max_size) + + memory_store.set_value(key, 1, "value") + + with pytest.raises(TimeIndexTooSmallOrNotExistingException): + memory_store.set_value(key, 0, "no_lower_index") + + +def test_can_update_exiting_times(key: str, memory_store: MemoryStore, max_size: int): + memory_store.init_collection(key, max_size) + + for k in range(10): + memory_store.set_value(key, 1, f"can_update_existing{k}") + + +def test_get_interval(fille_memory_store: MemoryStore, key: str): + values = fille_memory_store.get_interval(key, 1, 3) + assert values == [(1, "v1"), (2, "v2")] + + +def test_get_interval_missing_start_index(fille_memory_store: MemoryStore, key: str): + with pytest.raises( + TimeIndexMissingException, match="Provided index 100 was not found" + ): + fille_memory_store.get_interval(key, 100, 3) + + +def test_get_interval_missing_end_index(fille_memory_store: MemoryStore, key: str): + with pytest.raises( + TimeIndexMissingException, match="Provided index 200 was not found" + ): + fille_memory_store.get_interval(key, 1, 200) From b5fe0702ab19e86d37e4736404b53adcab38f302 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 13:57:52 +0100 Subject: [PATCH 02/86] added missing errors --- src/osparc_control/errors.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/osparc_control/errors.py diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py new file mode 100644 index 0000000..f3273ee --- /dev/null +++ b/src/osparc_control/errors.py @@ -0,0 +1,37 @@ +from collections import OrderedDict + + +class BaseControlException(Exception): + """inherited by all exceptions in this moduls""" + + +class CollectionIsFullException(BaseControlException): + """no more elements could be added to the list""" + + def __init__(self, key: str, max_items: int) -> None: + super().__init__(f"Current size of {key} exceeds {max_items}") + + +class KeyWasNotFoundException(BaseControlException): + def __init__(self, key: str) -> None: + super().__init__(f"Key {key} was not found") + + +class KeyIsAlreadyPresent(BaseControlException): + def __init__(self, key: str, data: OrderedDict) -> None: + super().__init__(f"Key {key} is already present with data {data}") + + +class TimeIndexTooSmallOrNotExistingException(BaseControlException): + def __init__(self, time_index: float, last_inserted_time_index: float) -> None: + message = ( + f"Trying to insert time={time_index} which is: " + f"grater than last inserted time={last_inserted_time_index} and " + f"not a previously existing value." + ) + super().__init__(message) + + +class TimeIndexMissingException(BaseControlException): + def __init__(self, time_index: float) -> None: + super().__init__(f"Provided index {time_index} was not found") From 2b102ea193ad554c8bf38e768b7da1e2614e86b0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 13:58:10 +0100 Subject: [PATCH 03/86] adding utils --- .gitignore | 1 + poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 487b123..f696d45 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /docs/_build/ /src/*.egg-info/ __pycache__/ +*ignore* \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index dec8059..930f6f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -731,6 +731,17 @@ python-versions = "*" [package.dependencies] docutils = ">=0.11,<1.0" +[[package]] +name = "rope" +version = "0.23.0" +description = "a python refactoring library..." +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +dev = ["build", "pytest", "pytest-timeout"] + [[package]] name = "ruamel.yaml" version = "0.17.21" @@ -1071,7 +1082,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "9f1008b794f0c162ba869bb7973817b2cb17649f710682a7a573b83a8685fa43" +content-hash = "5040c98c7df3ff363b7e802345a6103b357a92521a22e2fa8c09a6fdcf726f01" [metadata.files] alabaster = [ @@ -1512,6 +1523,10 @@ requests = [ restructuredtext-lint = [ {file = "restructuredtext_lint-1.4.0.tar.gz", hash = "sha256:1b235c0c922341ab6c530390892eb9e92f90b9b75046063e047cacfb0f050c45"}, ] +rope = [ + {file = "rope-0.23.0-py3-none-any.whl", hash = "sha256:edf2ed3c9b35a8814752ffd3ea55b293c791e5087e252461de898e953cf9c146"}, + {file = "rope-0.23.0.tar.gz", hash = "sha256:f87662c565086d660fc855cc07f37820267876634c3e9e51bddb32ff51547268"}, +] "ruamel.yaml" = [ {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, diff --git a/pyproject.toml b/pyproject.toml index 0b33974..66ecc72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ sphinx-click = "^3.0.2" Pygments = "^2.10.0" pyupgrade = "^2.29.1" furo = ">=2021.11.12" +rope = "^0.23.0" [tool.poetry.scripts] osparc-control = "osparc_control.__main__:main" From b75bdebb31d80860ed882e2599cf2d7524523c84 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 14:26:45 +0100 Subject: [PATCH 04/86] added basic Makefile --- Makefile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..40c6159 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +# Heps setup basic env development + +# poetry is required on your system +# suggested installation method +# or refer to official docs +# https://python-poetry.org/docs/ +.PHONY: install-poetry +install-poetry: + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + +# install deve dependecis as sugested by coockicutter +# https://cookiecutter-hypermodern-python.readthedocs.io/en/2021.11.26/quickstart.html +.PHONY: install-dev +install-dev: + pip install nox nox-poetry + +.PHONY: tests +tests: # run tests on lowest python interpreter + nox -r -s tests -p 3.6 + +.PHONY: nox-36 +nox-36: # runs nox with python 3.6 + nox -p 3.6 + +.PHONY: tests-dev +tests-dev: + pytest -vv -s --exitfirst --failed-first --pdb tests/ + +.PHONY: docs +docs: # runs and displays docs + #runs with py3.6 change the noxfile.py to use different interpreter version + nox -r -s docs From 99854f84364722b9c2b9e2210554768aa75a7ea6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 14:26:55 +0100 Subject: [PATCH 05/86] doc work with py 3.6 --- noxfile.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 4fc82e1..da25451 100644 --- a/noxfile.py +++ b/noxfile.py @@ -174,7 +174,7 @@ def xdoctest(session: Session) -> None: session.run("python", "-m", "xdoctest", *args) -@session(name="docs-build", python="3.10") +@session(name="docs-build", python="3.6") def docs_build(session: Session) -> None: """Build the documentation.""" args = session.posargs or ["docs", "docs/_build"] @@ -191,10 +191,16 @@ def docs_build(session: Session) -> None: session.run("sphinx-build", *args) -@session(python="3.10") +@session(python="3.6") def docs(session: Session) -> None: """Build and serve the documentation with live reloading on file changes.""" - args = session.posargs or ["--open-browser", "docs", "docs/_build"] + args = session.posargs or [ + "--host", + "0.0.0.0", + "--open-browser", + "docs", + "docs/_build", + ] session.install(".") session.install("sphinx", "sphinx-autobuild", "sphinx-click", "furo") From b4d84f614252e96ec3c6e43904d5c24e75e567f4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 14:50:25 +0100 Subject: [PATCH 06/86] trying to unlock CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0fc35b..c71c3c0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -112,7 +112,7 @@ jobs: path: docs/_build coverage: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: tests steps: - name: Check out the repository From f89645309f0230bd07d25161b4b7a05e79e3a48f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 14:52:40 +0100 Subject: [PATCH 07/86] reverting --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c71c3c0..c0fc35b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -112,7 +112,7 @@ jobs: path: docs/_build coverage: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: tests steps: - name: Check out the repository From 485d6ba6633e48fda6175bd2ad0cf0c0f4b4c1c6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Mar 2022 15:46:01 +0100 Subject: [PATCH 08/86] added in memory transport --- src/osparc_control/transport/in_memory.py | 29 +++++++++++++++++++++++ src/osparc_control/transport/interface.py | 11 +++++++++ tests/test_transport_in_memory.py | 28 ++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 src/osparc_control/transport/in_memory.py create mode 100644 src/osparc_control/transport/interface.py create mode 100644 tests/test_transport_in_memory.py diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py new file mode 100644 index 0000000..da8d236 --- /dev/null +++ b/src/osparc_control/transport/in_memory.py @@ -0,0 +1,29 @@ +from typing import Dict +from queue import Queue + +from .interface import BaseTransport + + +class InMemoryTransport(metaclass=BaseTransport): + """ + Blocking in memory implementation, working with queues. + Can only be mixed with threading. + + - sends data to `destination` + - fetches data from `source` + """ + + _SHARED_QUEUES: Dict[str, Queue] = {} + + def __init__(self, source: str, destination: str): + self.source: str = source + self.destination: str = destination + + self._SHARED_QUEUES[self.source] = Queue() + self._SHARED_QUEUES[self.destination] = Queue() + + def send_bytes(self, payload: bytes) -> None: + self._SHARED_QUEUES[self.destination].put(payload) + + def receive_bytes(self) -> bytes: + return self._SHARED_QUEUES[self.source].get() diff --git a/src/osparc_control/transport/interface.py b/src/osparc_control/transport/interface.py new file mode 100644 index 0000000..eccf310 --- /dev/null +++ b/src/osparc_control/transport/interface.py @@ -0,0 +1,11 @@ +from abc import ABCMeta, abstractmethod + + +class BaseTransport(ABCMeta): + @abstractmethod + def send_bytes(self, payload: bytes) -> None: + """sends bytes to remote""" + + @abstractmethod + def receive_bytes(self) -> bytes: + """returns bytes from remote""" diff --git a/tests/test_transport_in_memory.py b/tests/test_transport_in_memory.py new file mode 100644 index 0000000..81d8804 --- /dev/null +++ b/tests/test_transport_in_memory.py @@ -0,0 +1,28 @@ +from typing import Iterable +from osparc_control.transport.in_memory import InMemoryTransport + +# UTILS + + +def _payload_generator(start: int, stop: int) -> Iterable[bytes]: + assert start < stop + for k in range(start, stop): + yield f"test{k}".encode() + + +# TESTS + + +def test_send_receive(): + sender = InMemoryTransport("A", "B") + receiver = InMemoryTransport("B", "A") + + for message in _payload_generator(1, 10): + # TEST SENDER -> RECEIVER + sender.send_bytes(message) + assert receiver.receive_bytes() == message + + for message in _payload_generator(11, 20): + # TEST RECEIVER -> SENDER + receiver.send_bytes(message) + assert sender.receive_bytes() == message From 4d4f7fa30d8e1839ffe2dbd6d4ea085806327cf7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 14 Mar 2022 08:04:42 +0100 Subject: [PATCH 09/86] renamed --- .../transport/base_transport.py | 25 +++++++++++++++++++ src/osparc_control/transport/in_memory.py | 2 +- src/osparc_control/transport/interface.py | 11 -------- 3 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 src/osparc_control/transport/base_transport.py delete mode 100644 src/osparc_control/transport/interface.py diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py new file mode 100644 index 0000000..d0e8407 --- /dev/null +++ b/src/osparc_control/transport/base_transport.py @@ -0,0 +1,25 @@ +from abc import ABCMeta, abstractmethod + + +class BaseTransport(ABCMeta): + @abstractmethod + def send_bytes(self, payload: bytes) -> None: + """sends bytes to remote""" + + @abstractmethod + def receive_bytes(self) -> bytes: + """returns bytes from remote""" + + +class SenderReceiverPair: + """To be used by more custom protcols""" + + def __init__(self, sender: BaseTransport, receiver: BaseTransport) -> None: + self._sender: BaseTransport = sender + self._receiver: BaseTransport = receiver + + def send_bytes(self, message: bytes) -> None: + self._sender.send_bytes(message) + + def receive_bytes(self) -> bytes: + return self._receiver.receive_bytes() diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py index da8d236..ba88a63 100644 --- a/src/osparc_control/transport/in_memory.py +++ b/src/osparc_control/transport/in_memory.py @@ -1,7 +1,7 @@ from typing import Dict from queue import Queue -from .interface import BaseTransport +from .base_transport import BaseTransport class InMemoryTransport(metaclass=BaseTransport): diff --git a/src/osparc_control/transport/interface.py b/src/osparc_control/transport/interface.py deleted file mode 100644 index eccf310..0000000 --- a/src/osparc_control/transport/interface.py +++ /dev/null @@ -1,11 +0,0 @@ -from abc import ABCMeta, abstractmethod - - -class BaseTransport(ABCMeta): - @abstractmethod - def send_bytes(self, payload: bytes) -> None: - """sends bytes to remote""" - - @abstractmethod - def receive_bytes(self) -> bytes: - """returns bytes from remote""" From fab0aba8e67ca22f238b95c8d83a26344c8b9bae Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 14 Mar 2022 10:33:04 +0100 Subject: [PATCH 10/86] renaming --- src/osparc_control/memory.py | 2 +- tests/test_memory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osparc_control/memory.py b/src/osparc_control/memory.py index 4768e03..82cb6ca 100644 --- a/src/osparc_control/memory.py +++ b/src/osparc_control/memory.py @@ -53,7 +53,7 @@ def set_value(self, key: str, time_index: float, value: AcceptedValues) -> None: if len(self._store[key]) > self._max_sizes[key]: raise CollectionIsFullException(key=key, max_items=self._max_sizes[key]) - def size_of(self, key: str) -> int: + def size_of_collection(self, key: str) -> int: self._ensure_key(key) return len(self._store[key]) diff --git a/tests/test_memory.py b/tests/test_memory.py index bef034c..017d20f 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -66,7 +66,7 @@ def test_list_is_full(key: str, memory_store: MemoryStore, max_size: int): for t in range(max_size): memory_store.set_value(key, t, f"v{t}") - assert memory_store.size_of(key) == max_size + assert memory_store.size_of_collection(key) == max_size with pytest.raises(CollectionIsFullException): memory_store.set_value(key, 10, f"v10") From cc4b42233a2c2b2f7af8aea94b16982cebfb2205 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:23:05 +0100 Subject: [PATCH 11/86] add in_memory and base transport --- src/osparc_control/transport/__init__.py | 0 .../transport/base_transport.py | 27 ++++++++++++++++--- src/osparc_control/transport/in_memory.py | 14 +++++++--- tests/test_transport_in_memory.py | 6 +++++ 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 src/osparc_control/transport/__init__.py diff --git a/src/osparc_control/transport/__init__.py b/src/osparc_control/transport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index d0e8407..e7f2cf4 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -1,4 +1,5 @@ from abc import ABCMeta, abstractmethod +from typing import Optional class BaseTransport(ABCMeta): @@ -7,8 +8,19 @@ def send_bytes(self, payload: bytes) -> None: """sends bytes to remote""" @abstractmethod - def receive_bytes(self) -> bytes: - """returns bytes from remote""" + def receive_bytes(self) -> Optional[bytes]: + """ + returns bytes from remote + NOTE: this must never wait, it returns None if + nothing is avaliable + """ + + @abstractmethod + def thread_init(self) -> None: + """ + Some libraries require thread specific context. + This will be called by the thread once its started + """ class SenderReceiverPair: @@ -21,5 +33,14 @@ def __init__(self, sender: BaseTransport, receiver: BaseTransport) -> None: def send_bytes(self, message: bytes) -> None: self._sender.send_bytes(message) - def receive_bytes(self) -> bytes: + def receive_bytes(self) -> Optional[bytes]: + """this must never block""" return self._receiver.receive_bytes() + + def receiver_init(self) -> None: + """called by the background thread dealing with the receiver""" + self._receiver.thread_init() + + def sender_init(self) -> None: + """called by the background thread dealing with the sender""" + self._sender.thread_init() diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py index ba88a63..d6b5a8a 100644 --- a/src/osparc_control/transport/in_memory.py +++ b/src/osparc_control/transport/in_memory.py @@ -1,5 +1,5 @@ -from typing import Dict -from queue import Queue +from queue import Empty, Queue +from typing import Dict, Optional from .base_transport import BaseTransport @@ -25,5 +25,11 @@ def __init__(self, source: str, destination: str): def send_bytes(self, payload: bytes) -> None: self._SHARED_QUEUES[self.destination].put(payload) - def receive_bytes(self) -> bytes: - return self._SHARED_QUEUES[self.source].get() + def receive_bytes(self) -> Optional[bytes]: + try: + return self._SHARED_QUEUES[self.source].get(block=False) + except Empty: + return None + + def thread_init(self) -> None: + """no action required for this transport""" diff --git a/tests/test_transport_in_memory.py b/tests/test_transport_in_memory.py index 81d8804..a46eaf7 100644 --- a/tests/test_transport_in_memory.py +++ b/tests/test_transport_in_memory.py @@ -1,4 +1,5 @@ from typing import Iterable + from osparc_control.transport.in_memory import InMemoryTransport # UTILS @@ -26,3 +27,8 @@ def test_send_receive(): # TEST RECEIVER -> SENDER receiver.send_bytes(message) assert sender.receive_bytes() == message + + +def test_receive_returns_none_if_no_message_available(): + receiver = InMemoryTransport("B", "A") + assert receiver.receive_bytes() is None From 2cf73bfecb5e368281131a1fccf02fa68a3f052d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:33:04 +0100 Subject: [PATCH 12/86] fixed transport tests --- tests/test_transport.py | 47 +++++++++++++++++++++++++++++++ tests/test_transport_in_memory.py | 34 ---------------------- 2 files changed, 47 insertions(+), 34 deletions(-) create mode 100644 tests/test_transport.py delete mode 100644 tests/test_transport_in_memory.py diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..c99c0b9 --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,47 @@ +from typing import Iterable, Type + +import pytest + +from osparc_control.transport.base_transport import BaseTransport, SenderReceiverPair +from osparc_control.transport.in_memory import InMemoryTransport + +# UTILS + + +def _payload_generator(start: int, stop: int) -> Iterable[bytes]: + assert start < stop + for k in range(start, stop): + yield f"test{k}".encode() + + +# TESTS + + +@pytest.fixture(params=[InMemoryTransport]) +def transport_class(request) -> Type[BaseTransport]: + return request.param + + +def test_send_receive(transport_class: Type[BaseTransport]): + sender_receiver_pair = SenderReceiverPair( + sender=transport_class("A", "B"), + receiver=transport_class("B", "A"), + ) + + sender_receiver_pair.receiver_init() + sender_receiver_pair.sender_init() + + for message in _payload_generator(1, 10): + # TEST SENDER -> RECEIVER + sender_receiver_pair.send_bytes(message) + assert sender_receiver_pair.receive_bytes() == message + + for message in _payload_generator(11, 20): + # TEST RECEIVER -> SENDER + sender_receiver_pair.send_bytes(message) + assert sender_receiver_pair.receive_bytes() == message + + +def test_receive_returns_none_if_no_message_available(): + receiver = InMemoryTransport("B", "A") + assert receiver.receive_bytes() is None diff --git a/tests/test_transport_in_memory.py b/tests/test_transport_in_memory.py deleted file mode 100644 index a46eaf7..0000000 --- a/tests/test_transport_in_memory.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Iterable - -from osparc_control.transport.in_memory import InMemoryTransport - -# UTILS - - -def _payload_generator(start: int, stop: int) -> Iterable[bytes]: - assert start < stop - for k in range(start, stop): - yield f"test{k}".encode() - - -# TESTS - - -def test_send_receive(): - sender = InMemoryTransport("A", "B") - receiver = InMemoryTransport("B", "A") - - for message in _payload_generator(1, 10): - # TEST SENDER -> RECEIVER - sender.send_bytes(message) - assert receiver.receive_bytes() == message - - for message in _payload_generator(11, 20): - # TEST RECEIVER -> SENDER - receiver.send_bytes(message) - assert sender.receive_bytes() == message - - -def test_receive_returns_none_if_no_message_available(): - receiver = InMemoryTransport("B", "A") - assert receiver.receive_bytes() is None From 36d31fbc1c2b8165cdc943d19689fbc8a65e42f9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:33:13 +0100 Subject: [PATCH 13/86] format --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 9aa6e02..4770023 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,6 @@ """Sphinx configuration.""" from datetime import datetime - project = "Osparc Control" author = "Andrei Neagu" copyright = f"{datetime.now().year}, {author}" From d2afab03026c29252182f3a7a7823f084c183630 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:33:32 +0100 Subject: [PATCH 14/86] added models --- src/osparc_control/models.py | 111 +++++++++++++++++++++++++++++++++++ tests/test_models.py | 100 +++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 src/osparc_control/models.py create mode 100644 tests/test_models.py diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py new file mode 100644 index 0000000..1e6dd51 --- /dev/null +++ b/src/osparc_control/models.py @@ -0,0 +1,111 @@ +from enum import Enum +from typing import Any, Dict, List, Optional + +import umsgpack +from pydantic import BaseModel, Field, validator +from pyparsing import Opt + + +class _BaseSerializer(BaseModel): + def to_bytes(self) -> bytes: + return umsgpack.packb(self.dict()) + + @classmethod + def from_bytes(cls, raw: bytes) -> Optional[Any]: + return cls.parse_obj(umsgpack.unpackb(raw)) + + +class CommandParameter(BaseModel): + name: str = Field( + ..., description="name of the parameter to be provided with the commnad" + ) + description: str = Field( + ..., + description="provide more information to the user, how this command should be used", + ) + + +class CommnadType(str, Enum): + # NOTE: ANE -> KZ: can you please let me know if names and descriptions make sense? + # please suggest better ones + + # no reply is expected for this command + # nothing will be awaited + WITHOUT_REPLY = "WITHOUT_REPLY" + # a reply is expected and the user must check + # for the results + WITH_REPLAY = "WITH_REPLAY" + # user requests a parameter that he would like to have + # immediately, the request will be blocked until + # a value is returned + WAIT_FOR_REPLY = "WAIT_FOR_REPLY" + + +class CommandManifest(BaseModel): + action: str = Field(..., description="name of the action to be triggered on remote") + description: str = Field(..., description="more details about the action") + params: List[CommandParameter] = Field( + None, description="requested parameters by the user" + ) + command_type: CommnadType = Field( + ..., description="describes the command type, behaviour and usage" + ) + + @classmethod + def create( + cls, + action: str, + description: str, + command_type: CommnadType, + params: Optional[List[CommandParameter]] = None, + ) -> "CommandManifest": + """define a request which requires a reply and awaits for the reply""" + return cls( + action=action, + description=description, + command_type=command_type, + params=[] if params is None else params, + ) + + +class CommandRequest(_BaseSerializer): + request_id: str = Field(..., description="unique identifier") + action: str = Field(..., description="name of the action to be triggered on remote") + params: Dict[str, Any] = Field({}, description="requested parameters by the user") + command_type: CommnadType = Field( + ..., description="describes the command type, behaviour and usage" + ) + + @classmethod + def from_manifest( + cls, + manifest: CommandManifest, + request_id: str, + params: Optional[Dict[str, Any]] = None, + ) -> "CommandRequest": + params = {} if params is None else params + + if set(params.keys()) != {x.name for x in manifest.params}: + raise ValueError(f"Provided {params} did not match {manifest.params}") + + return cls( + request_id=request_id, + action=manifest.action, + params=params, + command_type=manifest.command_type, + ) + + +class CommandReply(_BaseSerializer): + reply_id: str = Field(..., description="unique identifier from request") + payload: Any = Field(..., description="user defined value for the command") + + +class TrackedRequest(BaseModel): + request: CommandRequest = Field(..., description="request being tracked") + reply: Optional[CommandReply] = Field( + None, description="reply will be not None if received" + ) + + +RequestsTracker = Dict[str, TrackedRequest] diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..40fed18 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,100 @@ +import json +from typing import Any, Dict, List, Optional + +import pytest +import umsgpack + +from osparc_control.models import ( + CommandManifest, + CommandParameter, + CommandRequest, + CommnadType, +) + + +@pytest.fixture +def request_id() -> str: + return "unique_id" + + +PARAMS: List[Optional[List[CommandParameter]]] = [ + None, + [], + [CommandParameter(name="first_arg", description="the first arg description")], +] + + +@pytest.mark.parametrize("params", PARAMS) +def test_command_manifest(params: Optional[List[CommandParameter]]): + for command_type in CommnadType: + assert CommandManifest.create( + action="test", + description="some test action", + command_type=command_type, + params=params, + ) + + +@pytest.mark.parametrize("params", PARAMS) +def test_command(request_id: str, params: Optional[List[CommandParameter]]): + request_params: Dict[str, Any] = ( + {} if params is None else {x.name: None for x in params} + ) + command = CommandManifest.create( + action="test", + description="some test action", + command_type=CommnadType.WITHOUT_REPLY, + params=params, + ) + + assert CommandRequest.from_manifest( + manifest=command, request_id=request_id, params=request_params + ) + + +@pytest.mark.parametrize("params", PARAMS) +def test_params_not_respecting_manifest( + request_id: str, params: Optional[List[CommandParameter]] +): + command = CommandManifest.create( + action="test", + description="some test action", + command_type=CommnadType.WAIT_FOR_REPLY, + params=params, + ) + + if params: + with pytest.raises(ValueError): + assert CommandRequest.from_manifest( + manifest=command, request_id=request_id, params={} + ) + else: + assert CommandRequest.from_manifest( + manifest=command, request_id=request_id, params={} + ) + + +@pytest.mark.parametrize("params", PARAMS) +def test_msgpack_serialization_deserialization( + request_id: str, params: Optional[List[CommandParameter]] +): + + request_params: Dict[str, Any] = ( + {} if params is None else {x.name: None for x in params} + ) + manifest = CommandManifest.create( + action="test", + description="some test action", + command_type=CommnadType.WAIT_FOR_REPLY, + params=params, + ) + + command_request = CommandRequest.from_manifest( + manifest=manifest, request_id=request_id, params=request_params + ) + + assert command_request == CommandRequest.from_bytes(command_request.to_bytes()) + + assert command_request.to_bytes() == umsgpack.packb( + json.loads(command_request.json()) + ) From 88f05f2d4aa672d27f66f6fa4ab6fec0dd346002 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:33:44 +0100 Subject: [PATCH 15/86] memory no longer used --- src/osparc_control/memory.py | 90 ----------------------- tests/test_memory.py | 134 ----------------------------------- 2 files changed, 224 deletions(-) delete mode 100644 src/osparc_control/memory.py delete mode 100644 tests/test_memory.py diff --git a/src/osparc_control/memory.py b/src/osparc_control/memory.py deleted file mode 100644 index 82cb6ca..0000000 --- a/src/osparc_control/memory.py +++ /dev/null @@ -1,90 +0,0 @@ -from typing import Dict, Union, List, Tuple -from collections import OrderedDict -from .types import AcceptedValues -from .errors import ( - CollectionIsFullException, - KeyIsAlreadyPresent, - KeyWasNotFoundException, - TimeIndexMissingException, - TimeIndexTooSmallOrNotExistingException, -) - - -class MemoryStore: - """Size limited time indexed memory store""" - - def __init__(self) -> None: - self._store: Dict[str, OrderedDict[float, AcceptedValues]] = {} - self._max_sizes: Dict[str, int] = {} - self._last_inserted_time_index: Dict[str, float] = {} - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self._store} {self._max_sizes}>" - - def _ensure_key(self, key: str) -> None: - if key not in self._store: - raise KeyWasNotFoundException(f"Key {key} was not found") - - def init_collection(self, key: str, max_size: int) -> None: - if key in self._store: - raise KeyIsAlreadyPresent(key, self._store[key]) - - self._store[key] = OrderedDict() - self._max_sizes[key] = max_size - - def set_value(self, key: str, time_index: float, value: AcceptedValues) -> None: - self._ensure_key(key) - - # ensure new previous times are not inserter - # it is ok to change their values but not to add new ones - last_inserted_time_index = self._last_inserted_time_index.get( - key, -float("inf") - ) - if last_inserted_time_index > time_index and time_index not in self._store[key]: - raise TimeIndexTooSmallOrNotExistingException( - time_index, last_inserted_time_index - ) - - self._store[key][time_index] = value - self._last_inserted_time_index[key] = time_index - - # collection is size limited, avoid memory blow up, user is required - # to specify the size before using it - if len(self._store[key]) > self._max_sizes[key]: - raise CollectionIsFullException(key=key, max_items=self._max_sizes[key]) - - def size_of_collection(self, key: str) -> int: - self._ensure_key(key) - return len(self._store[key]) - - def get_value( - self, key: str, time_index: float, default: Union[AcceptedValues, None] = None - ) -> Union[AcceptedValues, None]: - # maybe this should raise if time_index is missing? - self._ensure_key(key) - - return self._store[key].get(time_index, default) - - def get_interval( - self, key: str, time_index_start: float, time_index_stop: float - ) -> List[Tuple[float, AcceptedValues]]: - """works exactly like list slicing""" - self._ensure_key(key) - - collection: OrderedDict[float, AcceptedValues] = self._store[key] - - if time_index_start not in collection: - raise TimeIndexMissingException(time_index_start) - - if time_index_stop not in collection: - raise TimeIndexMissingException(time_index_stop) - - # map key_index to order than you search for them - key_to_index = {key: i for i, key in enumerate(collection.keys())} - - start_index = key_to_index[time_index_start] - stop_index = key_to_index[time_index_stop] - - collection_items = list(collection.items()) - - return collection_items[start_index:stop_index] diff --git a/tests/test_memory.py b/tests/test_memory.py deleted file mode 100644 index 017d20f..0000000 --- a/tests/test_memory.py +++ /dev/null @@ -1,134 +0,0 @@ -import pytest - -from osparc_control.memory import MemoryStore -from osparc_control.types import AcceptedValues -from osparc_control.errors import ( - CollectionIsFullException, - KeyIsAlreadyPresent, - KeyWasNotFoundException, - TimeIndexMissingException, - TimeIndexTooSmallOrNotExistingException, -) - - -@pytest.fixture -def key() -> str: - return "test_key" - - -@pytest.fixture -def max_size() -> int: - return 10 - - -@pytest.fixture -def memory_store() -> MemoryStore: - return MemoryStore() - - -@pytest.fixture -def fille_memory_store( - key: str, memory_store: MemoryStore, max_size: int -) -> MemoryStore: - memory_store.init_collection(key, max_size) - - for t in range(max_size): - memory_store.set_value(key, t, f"v{t}") - - return memory_store - - -def test_memory_store_init_set_get(key: str, memory_store: MemoryStore, max_size: int): - memory_store.init_collection(key, max_size) - - for t in range(max_size): - memory_store.set_value(key, t, f"v{t}") - - print(memory_store) - - assert memory_store.get_value(key, 0) == "v0" - assert memory_store.get_value(key, 9) == "v9" - - -def test_key_is_not_initialized(key: str, memory_store: MemoryStore): - with pytest.raises(KeyWasNotFoundException): - memory_store.get_value(key, 0) - - -def test_key_is_not_present(key: str, memory_store: MemoryStore): - with pytest.raises(KeyWasNotFoundException): - memory_store.set_value(key, 0, "10") - - -def test_list_is_full(key: str, memory_store: MemoryStore, max_size: int): - memory_store.init_collection(key, max_size) - - for t in range(max_size): - memory_store.set_value(key, t, f"v{t}") - - assert memory_store.size_of_collection(key) == max_size - - with pytest.raises(CollectionIsFullException): - memory_store.set_value(key, 10, f"v10") - - -def test_initialize_the_same_key_twice_fails( - key: str, memory_store: MemoryStore, max_size: int -): - memory_store.init_collection(key, max_size) - assert key in memory_store._store - assert key in memory_store._max_sizes - - with pytest.raises(KeyIsAlreadyPresent): - memory_store.init_collection(key, max_size) - - -def test_base_values(key: str, memory_store: MemoryStore, max_size: int): - memory_store.init_collection(key, max_size) - - def _assert_get_is_set(index: float, value: AcceptedValues) -> None: - memory_store.set_value(key, index, value) - stored = memory_store.get_value(key, index) - assert stored == value - - _assert_get_is_set(0, 1) - _assert_get_is_set(0, 1.0) - _assert_get_is_set(0, "1") - _assert_get_is_set(0, b"1") - - -def test_only_allow_increasing_times( - key: str, memory_store: MemoryStore, max_size: int -): - memory_store.init_collection(key, max_size) - - memory_store.set_value(key, 1, "value") - - with pytest.raises(TimeIndexTooSmallOrNotExistingException): - memory_store.set_value(key, 0, "no_lower_index") - - -def test_can_update_exiting_times(key: str, memory_store: MemoryStore, max_size: int): - memory_store.init_collection(key, max_size) - - for k in range(10): - memory_store.set_value(key, 1, f"can_update_existing{k}") - - -def test_get_interval(fille_memory_store: MemoryStore, key: str): - values = fille_memory_store.get_interval(key, 1, 3) - assert values == [(1, "v1"), (2, "v2")] - - -def test_get_interval_missing_start_index(fille_memory_store: MemoryStore, key: str): - with pytest.raises( - TimeIndexMissingException, match="Provided index 100 was not found" - ): - fille_memory_store.get_interval(key, 100, 3) - - -def test_get_interval_missing_end_index(fille_memory_store: MemoryStore, key: str): - with pytest.raises( - TimeIndexMissingException, match="Provided index 200 was not found" - ): - fille_memory_store.get_interval(key, 1, 200) From bbbd1d61a1b69619032a095be7079624f2210854 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:33:56 +0100 Subject: [PATCH 16/86] reformat --- noxfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index da25451..ed02bc1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,8 +8,7 @@ import nox try: - from nox_poetry import Session - from nox_poetry import session + from nox_poetry import Session, session except ImportError: message = f"""\ Nox failed to import the 'nox-poetry' package. From ab0972e8457e983d6f2218519704068110e69f13 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 11:53:13 +0100 Subject: [PATCH 17/86] added tests for commandreply --- src/osparc_control/models.py | 3 +-- src/osparc_control/types.py | 3 --- tests/test_models.py | 10 ++++++++++ 3 files changed, 11 insertions(+), 5 deletions(-) delete mode 100644 src/osparc_control/types.py diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 1e6dd51..0e68c9f 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -2,8 +2,7 @@ from typing import Any, Dict, List, Optional import umsgpack -from pydantic import BaseModel, Field, validator -from pyparsing import Opt +from pydantic import BaseModel, Field class _BaseSerializer(BaseModel): diff --git a/src/osparc_control/types.py b/src/osparc_control/types.py deleted file mode 100644 index 6d323c3..0000000 --- a/src/osparc_control/types.py +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Union - -AcceptedValues = Union[str, int, float, bytes] diff --git a/tests/test_models.py b/tests/test_models.py index 40fed18..1d5d7c4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -7,6 +7,7 @@ from osparc_control.models import ( CommandManifest, CommandParameter, + CommandReply, CommandRequest, CommnadType, ) @@ -98,3 +99,12 @@ def test_msgpack_serialization_deserialization( assert command_request.to_bytes() == umsgpack.packb( json.loads(command_request.json()) ) + + +@pytest.mark.parametrize("payload", [None, "a_string", 1, 1.0, b"some_bytes"]) +def test_command_reply_payloads_serialization_deserialization( + request_id: str, payload: Any +): + command_reply = CommandReply(reply_id=request_id, payload=payload) + assert command_reply + assert command_reply == CommandReply.from_bytes(command_reply.to_bytes()) From fdc0e9b02c1c6b07401a72b83da0ae80ea160390 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 15:06:07 +0100 Subject: [PATCH 18/86] added zmq base transport --- .../transport/base_transport.py | 37 ++++++++-- src/osparc_control/transport/in_memory.py | 15 +++- src/osparc_control/transport/zeromq.py | 70 +++++++++++++++++++ tests/test_transport.py | 51 ++++++++++---- 4 files changed, 149 insertions(+), 24 deletions(-) create mode 100644 src/osparc_control/transport/zeromq.py diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index e7f2cf4..407fba8 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -16,12 +16,29 @@ def receive_bytes(self) -> Optional[bytes]: """ @abstractmethod - def thread_init(self) -> None: + def sender_init(self) -> None: + """ + Some libraries require thread specific context. + This will be called by the thread once its started + """ + + @abstractmethod + def receiver_init(self) -> None: """ Some libraries require thread specific context. This will be called by the thread once its started """ + def sender_cleanup(self) -> None: + """ + Some libraries require cleanup when done with them + """ + + def receiver_cleanup(self) -> None: + """ + Some libraries require cleanup when done with them + """ + class SenderReceiverPair: """To be used by more custom protcols""" @@ -30,17 +47,23 @@ def __init__(self, sender: BaseTransport, receiver: BaseTransport) -> None: self._sender: BaseTransport = sender self._receiver: BaseTransport = receiver + def sender_init(self) -> None: + """called by the background thread dealing with the sender""" + self._sender.sender_init() + def send_bytes(self, message: bytes) -> None: self._sender.send_bytes(message) + def receiver_init(self) -> None: + """called by the background thread dealing with the receiver""" + self._receiver.receiver_init() + def receive_bytes(self) -> Optional[bytes]: """this must never block""" return self._receiver.receive_bytes() - def receiver_init(self) -> None: - """called by the background thread dealing with the receiver""" - self._receiver.thread_init() + def sender_cleanup(self) -> None: + self._sender.sender_cleanup() - def sender_init(self) -> None: - """called by the background thread dealing with the sender""" - self._sender.thread_init() + def receiver_cleanup(self) -> None: + self._receiver.receiver_cleanup() diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py index d6b5a8a..54c898d 100644 --- a/src/osparc_control/transport/in_memory.py +++ b/src/osparc_control/transport/in_memory.py @@ -6,7 +6,7 @@ class InMemoryTransport(metaclass=BaseTransport): """ - Blocking in memory implementation, working with queues. + Non blocking in memory implementation, working with queues. Can only be mixed with threading. - sends data to `destination` @@ -31,5 +31,14 @@ def receive_bytes(self) -> Optional[bytes]: except Empty: return None - def thread_init(self) -> None: - """no action required for this transport""" + def sender_init(self) -> None: + """no action required here""" + + def receiver_init(self) -> None: + """no action required here""" + + def sender_cleanup(self) -> None: + """no action required here""" + + def receiver_cleanup(self) -> None: + """no action required here""" diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py new file mode 100644 index 0000000..239fd0a --- /dev/null +++ b/src/osparc_control/transport/zeromq.py @@ -0,0 +1,70 @@ +from typing import Optional +from tenacity import RetryError, Retrying +from tenacity.stop import stop_after_attempt +from tenacity.wait import wait_fixed + +import zmq + +from .base_transport import BaseTransport + + +class ZeroMQTransport(metaclass=BaseTransport): + def __init__( + self, + listen_port: int, + remote_host: str, + remote_port: int, + ): + self.listen_port: int = listen_port + self.remote_host: str = remote_host + self.remote_port: int = remote_port + + self._recv_socket: Optional[zmq.socket.Socket] = None + self._send_socket: Optional[zmq.socket.Socket] = None + self._send_contex: Optional[zmq.context.Context] = None + self._recv_contex: Optional[zmq.context.Context] = None + + def send_bytes(self, payload: bytes) -> None: + assert self._send_socket + + self._send_socket.send(payload) + + def receive_bytes( + self, retry_count: int = 3, wait_between: float = 0.01 + ) -> Optional[bytes]: + assert self._recv_socket + + # try to fetch a message, usning unlocking sockets does not guarantee + # that data is always present, retry 3 times in a short amount of time + # this will guarantee the message arrives + try: + for attempt in Retrying( + stop=stop_after_attempt(retry_count), wait=wait_fixed(wait_between) + ): + with attempt: + message: bytes = self._recv_socket.recv(zmq.NOBLOCK) + return message + except RetryError: + return None + + def sender_init(self) -> None: + self._send_contex = zmq.Context() + self._send_socket = self._send_contex.socket(zmq.PUSH) + self._send_socket.bind(f"tcp://*:{self.listen_port}") + + def receiver_init(self) -> None: + self._recv_contex = zmq.Context() + self._recv_socket = self._recv_contex.socket(zmq.PULL) + self._recv_socket.connect(f"tcp://{self.remote_host}:{self.remote_port}") + + def sender_cleanup(self) -> None: + assert self._send_socket + self._send_socket.close() + assert self._send_contex + self._send_contex.term() + + def receiver_cleanup(self) -> None: + assert self._recv_socket + self._recv_socket.close() + assert self._recv_contex + self._recv_contex.term() diff --git a/tests/test_transport.py b/tests/test_transport.py index c99c0b9..55e57c7 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,9 +1,10 @@ from typing import Iterable, Type - +from threading import Thread import pytest from osparc_control.transport.base_transport import BaseTransport, SenderReceiverPair from osparc_control.transport.in_memory import InMemoryTransport +from osparc_control.transport.zeromq import ZeroMQTransport # UTILS @@ -17,29 +18,51 @@ def _payload_generator(start: int, stop: int) -> Iterable[bytes]: # TESTS -@pytest.fixture(params=[InMemoryTransport]) +@pytest.fixture(params=[InMemoryTransport, ZeroMQTransport]) def transport_class(request) -> Type[BaseTransport]: return request.param -def test_send_receive(transport_class: Type[BaseTransport]): - sender_receiver_pair = SenderReceiverPair( - sender=transport_class("A", "B"), - receiver=transport_class("B", "A"), - ) +@pytest.fixture +def sender_receiver_pair( + transport_class: Type[BaseTransport], +) -> Iterable[SenderReceiverPair]: + if transport_class == InMemoryTransport: + sender = transport_class("A", "B") + receiver = transport_class("B", "A") + elif transport_class == ZeroMQTransport: + port = 1111 + sender = transport_class( + listen_port=port, remote_host="localhost", remote_port=port + ) + receiver = transport_class( + listen_port=port, remote_host="localhost", remote_port=port + ) + + sender_receiver_pair = SenderReceiverPair(sender=sender, receiver=receiver) - sender_receiver_pair.receiver_init() sender_receiver_pair.sender_init() + sender_receiver_pair.receiver_init() + + yield sender_receiver_pair + + sender_receiver_pair.sender_cleanup() + sender_receiver_pair.receiver_cleanup() + +def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair): for message in _payload_generator(1, 10): - # TEST SENDER -> RECEIVER + print("sending", message) sender_receiver_pair.send_bytes(message) - assert sender_receiver_pair.receive_bytes() == message - for message in _payload_generator(11, 20): - # TEST RECEIVER -> SENDER - sender_receiver_pair.send_bytes(message) - assert sender_receiver_pair.receive_bytes() == message + for expected_message in _payload_generator(1, 10): + message = sender_receiver_pair.receive_bytes() + print("received", message) + assert message == expected_message + + +def test_receive_nothing(sender_receiver_pair: SenderReceiverPair): + assert sender_receiver_pair.receive_bytes() == None def test_receive_returns_none_if_no_message_available(): From bed07249e42ea9fe2bf8fad69598be4c7e63c293 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 16:47:44 +0100 Subject: [PATCH 19/86] first base working version --- src/osparc_control/core.py | 241 +++++++++++++++++++++++++ src/osparc_control/errors.py | 35 +--- src/osparc_control/models.py | 11 +- src/osparc_control/transport/zeromq.py | 7 +- 4 files changed, 251 insertions(+), 43 deletions(-) create mode 100644 src/osparc_control/core.py diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py new file mode 100644 index 0000000..cdc8ed1 --- /dev/null +++ b/src/osparc_control/core.py @@ -0,0 +1,241 @@ +from collections import deque +from queue import Queue +from threading import Thread +from time import sleep +from typing import Any, Dict, Optional, Tuple, Union +from uuid import uuid4, getnode +from pydantic import ValidationError + +from tenacity import RetryError, Retrying +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_fixed + +from osparc_control.transport.zeromq import ZeroMQTransport + +from .errors import NoReplyException +from .models import ( + CommandManifest, + CommandReply, + CommandRequest, + CommnadType, + RequestsTracker, + CommandBase, + TrackedRequest, +) +from .transport.base_transport import SenderReceiverPair + +WAIT_FOR_MESSAGES: float = 0.01 +WAIT_BETWEEN_CHECKS: float = 0.1 + + +def _generate_request_id() -> str: + unique_hardware_id: int = getnode() + return f"{unique_hardware_id}_{uuid4()}" + + +def _get_sender_receiver_pair( + listen_port: int, remote_host: str, remote_port: int +) -> SenderReceiverPair: + sender = ZeroMQTransport( + listen_port=listen_port, remote_host=remote_host, remote_port=remote_port + ) + receiver = ZeroMQTransport( + listen_port=listen_port, remote_host=remote_host, remote_port=remote_port + ) + return SenderReceiverPair(sender=sender, receiver=receiver) + + +class SomeEntryPoint: + def __init__( + self, remote_host: str, remote_port: int = 7426, listen_port: int = 7426 + ) -> None: + + self._sender_receiver_pair: SenderReceiverPair = _get_sender_receiver_pair( + remote_host=remote_host, remote_port=remote_port, listen_port=listen_port + ) + + self._request_tracker: RequestsTracker = {} + # TODO: below must be protected by a lock + self._incoming_request_tracker = deque() + + self._out_queue: Queue = Queue() + + # sending and receving threads + self._sender_thread: Thread = Thread(target=self._sender_worker) + self._receiver_thread: Thread = Thread(target=self._receiver_worker) + self._continue: bool = True + + def _sender_worker(self) -> None: + self._sender_receiver_pair.sender_init() + + while self._continue: + message = self._out_queue.get() + message: Optional[Union[CommandRequest, CommandReply]] = message + if message is None: + # exit worker + break + + print("Message to deliver", message) + # send message + self._sender_receiver_pair.send_bytes(message.to_bytes()) + + self._sender_receiver_pair.sender_cleanup() + + def _receiver_worker(self) -> None: + self._sender_receiver_pair.receiver_init() + + while self._continue: + # this is blocking should be under timeout block + response: Optional[bytes] = self._sender_receiver_pair.receive_bytes() + if response is None: + # no messages available + continue + + # NOTE: pydantic does not support polymorphism + # SEE https://github.com/samuelcolvin/pydantic/issues/503 + + # check if is CommandRequest + try: + command_request = CommandRequest.from_bytes(response) + assert command_request + self._incoming_request_tracker.append(command_request) + except ValidationError: + pass + + # check if is CommandReply + try: + command_reply = CommandReply.from_bytes(response) + assert command_reply + tracked_request: TrackedRequest = self._request_tracker[ + command_reply.reply_id + ] + tracked_request.reply = command_reply + except ValidationError: + pass + + sleep(WAIT_FOR_MESSAGES) + + self._sender_receiver_pair.receiver_cleanup() + + def _enqueue_call( + self, + manifest: CommandManifest, + params: Optional[Dict[str, Any]], + expected_command_type: CommnadType, + ) -> CommandRequest: + """validates and enqueues the call for delivery to remote""" + request = CommandRequest.from_manifest( + manifest=manifest, request_id=_generate_request_id(), params=params + ) + + if request.command_type != expected_command_type: + raise RuntimeError( + f"Request {request} was expected to have command_type={expected_command_type}" + ) + + self._request_tracker[request.request_id] = TrackedRequest( + request=request, reply=None + ) + + self._out_queue.put(request) + + return request + + def start_background_sync(self) -> None: + """starts workers handling data transfer""" + self._continue = True + self._sender_thread.start() + self._receiver_thread.start() + + def stop_background_sync(self) -> None: + """stops workers handling data transfer""" + self._continue = False + + # stopping workers + self._out_queue.put(None) + + self._sender_thread.join() + self._receiver_thread.join() + + def request_and_forget( + self, manifest: CommandManifest, params: Optional[Dict[str, Any]] = None + ) -> None: + """No reply will be provided by remote side for this command""" + self._enqueue_call(manifest, params, CommnadType.WITHOUT_REPLY) + + def request_and_check( + self, manifest: CommandManifest, params: Optional[Dict[str, Any]] = None + ) -> str: + """ + returns a `request_id` to be used with `check_for_reply` to monitor + if a reply to the request was returned. + """ + request = self._enqueue_call(manifest, params, CommnadType.WITH_REPLAY) + return request.request_id + + def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: + """ + Checks if a reply to for the request_id provided by `request_and_check` + is available. + + returns a tuple where: + - first entry is True if the reply to the request was returned + - second element is the actual returned value of the reply + """ + tracked_request: Optional[TrackedRequest] = self._request_tracker.get( + request_id, None + ) + if tracked_request is None: + return False, None + + # check for the correct type of request + if not tracked_request.request.command_type != CommnadType.WAIT_FOR_REPLY: + raise RuntimeError( + ( + f"Request {tracked_request.request} not expect a " + f"reply, found reply {tracked_request.reply}" + ) + ) + + # check if reply was received + if tracked_request.reply is None: + return False, None + + return True, tracked_request.reply.payload + + def request_and_wait( + self, + manifest: CommandManifest, + timeout: float, + params: Optional[Dict[str, Any]] = None, + ) -> Optional[Any]: + """ + Requests and awaits for the response from remote. + A timeout for this function is required. If the timeout is reached `None` will + be returned. + """ + request = self._enqueue_call(manifest, params, CommnadType.WAIT_FOR_REPLY) + + try: + for attempt in Retrying( + stop=stop_after_delay(timeout), wait=wait_fixed(WAIT_BETWEEN_CHECKS) + ): + with attempt: + reply_received, result = self.check_for_reply(request.request_id) + if not reply_received: + raise NoReplyException() + + return result + except RetryError: + return None + + def get_incoming_request(self) -> Optional[CommandRequest]: + """will try to fetch an incoming request, returns None if nothing is present""" + try: + return self._incoming_request_tracker.pop() + except IndexError: + return None + + def reply_to_command(self, request_id: str, payload: Any) -> None: + """provide the reply back to a command""" + self._out_queue.put(CommandReply(reply_id=request_id, payload=payload)) diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index f3273ee..5b0b197 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -1,37 +1,6 @@ -from collections import OrderedDict - - class BaseControlException(Exception): """inherited by all exceptions in this moduls""" -class CollectionIsFullException(BaseControlException): - """no more elements could be added to the list""" - - def __init__(self, key: str, max_items: int) -> None: - super().__init__(f"Current size of {key} exceeds {max_items}") - - -class KeyWasNotFoundException(BaseControlException): - def __init__(self, key: str) -> None: - super().__init__(f"Key {key} was not found") - - -class KeyIsAlreadyPresent(BaseControlException): - def __init__(self, key: str, data: OrderedDict) -> None: - super().__init__(f"Key {key} is already present with data {data}") - - -class TimeIndexTooSmallOrNotExistingException(BaseControlException): - def __init__(self, time_index: float, last_inserted_time_index: float) -> None: - message = ( - f"Trying to insert time={time_index} which is: " - f"grater than last inserted time={last_inserted_time_index} and " - f"not a previously existing value." - ) - super().__init__(message) - - -class TimeIndexMissingException(BaseControlException): - def __init__(self, time_index: float) -> None: - super().__init__(f"Provided index {time_index} was not found") +class NoReplyException(BaseControlException): + """Used when retrying for a result""" diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 0e68c9f..ca1e229 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -2,10 +2,10 @@ from typing import Any, Dict, List, Optional import umsgpack -from pydantic import BaseModel, Field +from pydantic import BaseModel, Extra, Field -class _BaseSerializer(BaseModel): +class CommandBase(BaseModel): def to_bytes(self) -> bytes: return umsgpack.packb(self.dict()) @@ -13,6 +13,9 @@ def to_bytes(self) -> bytes: def from_bytes(cls, raw: bytes) -> Optional[Any]: return cls.parse_obj(umsgpack.unpackb(raw)) + class Config: + extra = Extra.allow + class CommandParameter(BaseModel): name: str = Field( @@ -67,7 +70,7 @@ def create( ) -class CommandRequest(_BaseSerializer): +class CommandRequest(CommandBase): request_id: str = Field(..., description="unique identifier") action: str = Field(..., description="name of the action to be triggered on remote") params: Dict[str, Any] = Field({}, description="requested parameters by the user") @@ -95,7 +98,7 @@ def from_manifest( ) -class CommandReply(_BaseSerializer): +class CommandReply(CommandBase): reply_id: str = Field(..., description="unique identifier from request") payload: Any = Field(..., description="user defined value for the command") diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py index 239fd0a..85b1a7c 100644 --- a/src/osparc_control/transport/zeromq.py +++ b/src/osparc_control/transport/zeromq.py @@ -9,12 +9,7 @@ class ZeroMQTransport(metaclass=BaseTransport): - def __init__( - self, - listen_port: int, - remote_host: str, - remote_port: int, - ): + def __init__(self, listen_port: int, remote_host: str, remote_port: int): self.listen_port: int = listen_port self.remote_host: str = remote_host self.remote_port: int = remote_port From a78d55772e906000b1b0bfa68370a86442cc177f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 16:48:06 +0100 Subject: [PATCH 20/86] updated dependencies --- poetry.lock | 222 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 4 + 2 files changed, 223 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 930f6f9..030e951 100644 --- a/poetry.lock +++ b/poetry.lock @@ -119,6 +119,17 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "cffi" +version = "1.15.0" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfgv" version = "3.3.1" @@ -184,7 +195,7 @@ python-versions = ">=3.6,<4.0" name = "dataclasses" version = "0.8" description = "A backport of the dataclasses module for Python 3.6" -category = "dev" +category = "main" optional = false python-versions = ">=3.6, <3.7" @@ -589,7 +600,7 @@ toml = "*" name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -601,6 +612,30 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pydantic" +version = "1.9.0" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pydocstyle" version = "6.1.1" @@ -691,6 +726,18 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "pyzmq" +version = "22.3.0" +description = "Python bindings for 0MQ" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} +py = {version = "*", markers = "implementation_name == \"pypy\""} + [[package]] name = "reorder-python-imports" version = "2.6.0" @@ -954,6 +1001,17 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} pbr = ">=2.0.0,<2.1.0 || >2.1.0" +[[package]] +name = "tenacity" +version = "8.0.1" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "tokenize-rt" version = "4.2.1" @@ -1014,6 +1072,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "u-msgpack-python" +version = "2.7.1" +description = "A portable, lightweight MessagePack serializer and deserializer written in pure Python." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "urllib3" version = "1.26.8" @@ -1082,7 +1148,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "5040c98c7df3ff363b7e802345a6103b357a92521a22e2fa8c09a6fdcf726f01" +content-hash = "8c9ff7190cd2d985bff1a6636e8f54011318ae3afee70ede8e37a8a3504b1e78" [metadata.files] alabaster = [ @@ -1146,6 +1212,58 @@ certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] +cffi = [ + {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, + {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, + {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, + {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, + {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, + {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, + {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, + {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, + {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, + {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, + {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, + {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, + {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, + {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, + {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, + {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, + {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, +] cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, @@ -1449,6 +1567,47 @@ pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, ] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pydantic = [ + {file = "pydantic-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb23bcc093697cdea2708baae4f9ba0e972960a835af22560f6ae4e7e47d33f5"}, + {file = "pydantic-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d5278bd9f0eee04a44c712982343103bba63507480bfd2fc2790fa70cd64cf4"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab624700dc145aa809e6f3ec93fb8e7d0f99d9023b713f6a953637429b437d37"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d7da6f1c1049eefb718d43d99ad73100c958a5367d30b9321b092771e96c25"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c3b035103bd4e2e4a28da9da7ef2fa47b00ee4a9cf4f1a735214c1bcd05e0f6"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3011b975c973819883842c5ab925a4e4298dffccf7782c55ec3580ed17dc464c"}, + {file = "pydantic-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:086254884d10d3ba16da0588604ffdc5aab3f7f09557b998373e885c690dd398"}, + {file = "pydantic-1.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0fe476769acaa7fcddd17cadd172b156b53546ec3614a4d880e5d29ea5fbce65"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8e9dcf1ac499679aceedac7e7ca6d8641f0193c591a2d090282aaf8e9445a46"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e4c28f30e767fd07f2ddc6f74f41f034d1dd6bc526cd59e63a82fe8bb9ef4c"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c86229333cabaaa8c51cf971496f10318c4734cf7b641f08af0a6fbf17ca3054"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c0727bda6e38144d464daec31dff936a82917f431d9c39c39c60a26567eae3ed"}, + {file = "pydantic-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:dee5ef83a76ac31ab0c78c10bd7d5437bfdb6358c95b91f1ba7ff7b76f9996a1"}, + {file = "pydantic-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9c9bdb3af48e242838f9f6e6127de9be7063aad17b32215ccc36a09c5cf1070"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee7e3209db1e468341ef41fe263eb655f67f5c5a76c924044314e139a1103a2"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b6037175234850ffd094ca77bf60fb54b08b5b22bc85865331dd3bda7a02fa1"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b2571db88c636d862b35090ccf92bf24004393f85c8870a37f42d9f23d13e032"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b5ac0f1c83d31b324e57a273da59197c83d1bb18171e512908fe5dc7278a1d6"}, + {file = "pydantic-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bbbc94d0c94dd80b3340fc4f04fd4d701f4b038ebad72c39693c794fd3bc2d9d"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e0896200b6a40197405af18828da49f067c2fa1f821491bc8f5bde241ef3f7d7"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bdfdadb5994b44bd5579cfa7c9b0e1b0e540c952d56f627eb227851cda9db77"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:574936363cd4b9eed8acdd6b80d0143162f2eb654d96cb3a8ee91d3e64bf4cf9"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c556695b699f648c58373b542534308922c46a1cda06ea47bc9ca45ef5b39ae6"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f947352c3434e8b937e3aa8f96f47bdfe6d92779e44bb3f41e4c213ba6a32145"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5e48ef4a8b8c066c4a31409d91d7ca372a774d0212da2787c0d32f8045b1e034"}, + {file = "pydantic-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:96f240bce182ca7fe045c76bcebfa0b0534a1bf402ed05914a6f1dadff91877f"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:815ddebb2792efd4bba5488bc8fde09c29e8ca3227d27cf1c6990fc830fd292b"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c5b77947b9e85a54848343928b597b4f74fc364b70926b3c4441ff52620640c"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c68c3bc88dbda2a6805e9a142ce84782d3930f8fdd9655430d8576315ad97ce"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a79330f8571faf71bf93667d3ee054609816f10a259a109a0738dac983b23c3"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5a64b64ddf4c99fe201ac2724daada8595ada0d102ab96d019c1555c2d6441d"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a733965f1a2b4090a5238d40d983dcd78f3ecea221c7af1497b845a9709c1721"}, + {file = "pydantic-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cc6a4cb8a118ffec2ca5fcb47afbacb4f16d0ab8b7350ddea5e8ef7bcc53a16"}, + {file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"}, + {file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"}, +] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, @@ -1512,6 +1671,55 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +pyzmq = [ + {file = "pyzmq-22.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:6b217b8f9dfb6628f74b94bdaf9f7408708cb02167d644edca33f38746ca12dd"}, + {file = "pyzmq-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2841997a0d85b998cbafecb4183caf51fd19c4357075dfd33eb7efea57e4c149"}, + {file = "pyzmq-22.3.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f89468059ebc519a7acde1ee50b779019535db8dcf9b8c162ef669257fef7a93"}, + {file = "pyzmq-22.3.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea12133df25e3a6918718fbb9a510c6ee5d3fdd5a346320421aac3882f4feeea"}, + {file = "pyzmq-22.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76c532fd68b93998aab92356be280deec5de8f8fe59cd28763d2cc8a58747b7f"}, + {file = "pyzmq-22.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f907c7359ce8bf7f7e63c82f75ad0223384105f5126f313400b7e8004d9b33c3"}, + {file = "pyzmq-22.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:902319cfe23366595d3fa769b5b751e6ee6750a0a64c5d9f757d624b2ac3519e"}, + {file = "pyzmq-22.3.0-cp310-cp310-win32.whl", hash = "sha256:67db33bea0a29d03e6eeec55a8190e033318cee3cbc732ba8fd939617cbf762d"}, + {file = "pyzmq-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:7661fc1d5cb73481cf710a1418a4e1e301ed7d5d924f91c67ba84b2a1b89defd"}, + {file = "pyzmq-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79244b9e97948eaf38695f4b8e6fc63b14b78cc37f403c6642ba555517ac1268"}, + {file = "pyzmq-22.3.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab888624ed68930442a3f3b0b921ad7439c51ba122dbc8c386e6487a658e4a4e"}, + {file = "pyzmq-22.3.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18cd854b423fce44951c3a4d3e686bac8f1243d954f579e120a1714096637cc0"}, + {file = "pyzmq-22.3.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:de8df0684398bd74ad160afdc2a118ca28384ac6f5e234eb0508858d8d2d9364"}, + {file = "pyzmq-22.3.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:62bcade20813796c426409a3e7423862d50ff0639f5a2a95be4b85b09a618666"}, + {file = "pyzmq-22.3.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ea5a79e808baef98c48c884effce05c31a0698c1057de8fc1c688891043c1ce1"}, + {file = "pyzmq-22.3.0-cp36-cp36m-win32.whl", hash = "sha256:3c1895c95be92600233e476fe283f042e71cf8f0b938aabf21b7aafa62a8dac9"}, + {file = "pyzmq-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:851977788b9caa8ed011f5f643d3ee8653af02c5fc723fa350db5125abf2be7b"}, + {file = "pyzmq-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b4ebed0977f92320f6686c96e9e8dd29eed199eb8d066936bac991afc37cbb70"}, + {file = "pyzmq-22.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42abddebe2c6a35180ca549fadc7228d23c1e1f76167c5ebc8a936b5804ea2df"}, + {file = "pyzmq-22.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1e41b32d6f7f9c26bc731a8b529ff592f31fc8b6ef2be9fa74abd05c8a342d7"}, + {file = "pyzmq-22.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:be4e0f229cf3a71f9ecd633566bd6f80d9fa6afaaff5489492be63fe459ef98c"}, + {file = "pyzmq-22.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08c4e315a76ef26eb833511ebf3fa87d182152adf43dedee8d79f998a2162a0b"}, + {file = "pyzmq-22.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:badb868fff14cfd0e200eaa845887b1011146a7d26d579aaa7f966c203736b92"}, + {file = "pyzmq-22.3.0-cp37-cp37m-win32.whl", hash = "sha256:7c58f598d9fcc52772b89a92d72bf8829c12d09746a6d2c724c5b30076c1f11d"}, + {file = "pyzmq-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2b97502c16a5ec611cd52410bdfaab264997c627a46b0f98d3f666227fd1ea2d"}, + {file = "pyzmq-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d728b08448e5ac3e4d886b165385a262883c34b84a7fe1166277fe675e1c197a"}, + {file = "pyzmq-22.3.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:480b9931bfb08bf8b094edd4836271d4d6b44150da051547d8c7113bf947a8b0"}, + {file = "pyzmq-22.3.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7dc09198e4073e6015d9a8ea093fc348d4e59de49382476940c3dd9ae156fba8"}, + {file = "pyzmq-22.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ca6cd58f62a2751728016d40082008d3b3412a7f28ddfb4a2f0d3c130f69e74"}, + {file = "pyzmq-22.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:468bd59a588e276961a918a3060948ae68f6ff5a7fa10bb2f9160c18fe341067"}, + {file = "pyzmq-22.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c88fa7410e9fc471e0858638f403739ee869924dd8e4ae26748496466e27ac59"}, + {file = "pyzmq-22.3.0-cp38-cp38-win32.whl", hash = "sha256:c0f84360dcca3481e8674393bdf931f9f10470988f87311b19d23cda869bb6b7"}, + {file = "pyzmq-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:f762442bab706fd874064ca218b33a1d8e40d4938e96c24dafd9b12e28017f45"}, + {file = "pyzmq-22.3.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:954e73c9cd4d6ae319f1c936ad159072b6d356a92dcbbabfd6e6204b9a79d356"}, + {file = "pyzmq-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f43b4a2e6218371dd4f41e547bd919ceeb6ebf4abf31a7a0669cd11cd91ea973"}, + {file = "pyzmq-22.3.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:acebba1a23fb9d72b42471c3771b6f2f18dcd46df77482612054bd45c07dfa36"}, + {file = "pyzmq-22.3.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cf98fd7a6c8aaa08dbc699ffae33fd71175696d78028281bc7b832b26f00ca57"}, + {file = "pyzmq-22.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d072f7dfbdb184f0786d63bda26e8a0882041b1e393fbe98940395f7fab4c5e2"}, + {file = "pyzmq-22.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:53f4fd13976789ffafedd4d46f954c7bb01146121812b72b4ddca286034df966"}, + {file = "pyzmq-22.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1b5d457acbadcf8b27561deeaa386b0217f47626b29672fa7bd31deb6e91e1b"}, + {file = "pyzmq-22.3.0-cp39-cp39-win32.whl", hash = "sha256:e6a02cf7271ee94674a44f4e62aa061d2d049001c844657740e156596298b70b"}, + {file = "pyzmq-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d3dcb5548ead4f1123851a5ced467791f6986d68c656bc63bfff1bf9e36671e2"}, + {file = "pyzmq-22.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3a4c9886d61d386b2b493377d980f502186cd71d501fffdba52bd2a0880cef4f"}, + {file = "pyzmq-22.3.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:80e043a89c6cadefd3a0712f8a1322038e819ebe9dbac7eca3bce1721bcb63bf"}, + {file = "pyzmq-22.3.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1621e7a2af72cced1f6ec8ca8ca91d0f76ac236ab2e8828ac8fe909512d566cb"}, + {file = "pyzmq-22.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d6157793719de168b199194f6b6173f0ccd3bf3499e6870fac17086072e39115"}, + {file = "pyzmq-22.3.0.tar.gz", hash = "sha256:8eddc033e716f8c91c6a2112f0a8ebc5e00532b4a6ae1eb0ccc48e027f9c671c"}, +] reorder-python-imports = [ {file = "reorder_python_imports-2.6.0-py2.py3-none-any.whl", hash = "sha256:54a3afd594a3959b10f7eb8b54ef453eb2b5176eb7b01c111cb1893ff9a2c685"}, {file = "reorder_python_imports-2.6.0.tar.gz", hash = "sha256:f4dc03142bdb57625e64299aea80e9055ce0f8b719f8f19c217a487c9fa9379e"}, @@ -1618,6 +1826,10 @@ stevedore = [ {file = "stevedore-3.5.0-py3-none-any.whl", hash = "sha256:a547de73308fd7e90075bb4d301405bebf705292fa90a90fc3bcf9133f58616c"}, {file = "stevedore-3.5.0.tar.gz", hash = "sha256:f40253887d8712eaa2bb0ea3830374416736dc8ec0e22f5a65092c1174c44335"}, ] +tenacity = [ + {file = "tenacity-8.0.1-py3-none-any.whl", hash = "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a"}, + {file = "tenacity-8.0.1.tar.gz", hash = "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f"}, +] tokenize-rt = [ {file = "tokenize_rt-4.2.1-py2.py3-none-any.whl", hash = "sha256:08a27fa032a81cf45e8858d0ac706004fcd523e8463415ddf1442be38e204ea8"}, {file = "tokenize_rt-4.2.1.tar.gz", hash = "sha256:0d4f69026fed520f8a1e0103aa36c406ef4661417f20ca643f913e33531b3b94"}, @@ -1713,6 +1925,10 @@ typing-extensions = [ {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] +u-msgpack-python = [ + {file = "u-msgpack-python-2.7.1.tar.gz", hash = "sha256:b7e7d433cab77171a4c752875d91836f3040306bab5063fb6dbe11f64ea69551"}, + {file = "u_msgpack_python-2.7.1-py2.py3-none-any.whl", hash = "sha256:0eb339ae27ec3085945244d17b74fd1ed875e866974d63caaa85d90fca9060a7"}, +] urllib3 = [ {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, diff --git a/pyproject.toml b/pyproject.toml index 66ecc72..29d36a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,10 @@ Changelog = "https://github.com/ITISFoundation/osparc-control/releases" [tool.poetry.dependencies] python = "^3.6.2" click = "^8.0.1" +pyzmq = ">=14.0.0" +u-msgpack-python = ">=2.7.1" +pydantic = ">=1.8.2" +tenacity = ">=8.0.1" [tool.poetry.dev-dependencies] pytest = "^6.2.5" From c07e06ef2c76a867947ace838f768071b3b43fb1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 15 Mar 2022 16:49:59 +0100 Subject: [PATCH 21/86] codestyle --- src/osparc_control/core.py | 5 ++--- src/osparc_control/transport/zeromq.py | 4 ++-- tests/test_transport.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index cdc8ed1..4f581f3 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -3,9 +3,9 @@ from threading import Thread from time import sleep from typing import Any, Dict, Optional, Tuple, Union -from uuid import uuid4, getnode -from pydantic import ValidationError +from uuid import getnode, uuid4 +from pydantic import ValidationError from tenacity import RetryError, Retrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed @@ -19,7 +19,6 @@ CommandRequest, CommnadType, RequestsTracker, - CommandBase, TrackedRequest, ) from .transport.base_transport import SenderReceiverPair diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py index 85b1a7c..ebb9f57 100644 --- a/src/osparc_control/transport/zeromq.py +++ b/src/osparc_control/transport/zeromq.py @@ -1,10 +1,10 @@ from typing import Optional + +import zmq from tenacity import RetryError, Retrying from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed -import zmq - from .base_transport import BaseTransport diff --git a/tests/test_transport.py b/tests/test_transport.py index 55e57c7..52dd47a 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,5 +1,5 @@ from typing import Iterable, Type -from threading import Thread + import pytest from osparc_control.transport.base_transport import BaseTransport, SenderReceiverPair From ff2e3d20050ac6970fa464ab703b27204624604d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 08:26:20 +0100 Subject: [PATCH 22/86] refactor models --- src/osparc_control/models.py | 46 +++++++------------------------- tests/test_models.py | 51 ++++++++++++++---------------------- 2 files changed, 29 insertions(+), 68 deletions(-) diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index ca1e229..bc27aa6 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -1,5 +1,7 @@ +from cgitb import handler from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional +from xml.dom import InvalidAccessErr import umsgpack from pydantic import BaseModel, Extra, Field @@ -52,22 +54,13 @@ class CommandManifest(BaseModel): command_type: CommnadType = Field( ..., description="describes the command type, behaviour and usage" ) - - @classmethod - def create( - cls, - action: str, - description: str, - command_type: CommnadType, - params: Optional[List[CommandParameter]] = None, - ) -> "CommandManifest": - """define a request which requires a reply and awaits for the reply""" - return cls( - action=action, - description=description, - command_type=command_type, - params=[] if params is None else params, - ) + handler: Optional[Callable] = Field( + None, + description=( + "if the user provides a callable it will be called to handle" + "incoming requests" + ), + ) class CommandRequest(CommandBase): @@ -78,25 +71,6 @@ class CommandRequest(CommandBase): ..., description="describes the command type, behaviour and usage" ) - @classmethod - def from_manifest( - cls, - manifest: CommandManifest, - request_id: str, - params: Optional[Dict[str, Any]] = None, - ) -> "CommandRequest": - params = {} if params is None else params - - if set(params.keys()) != {x.name for x in manifest.params}: - raise ValueError(f"Provided {params} did not match {manifest.params}") - - return cls( - request_id=request_id, - action=manifest.action, - params=params, - command_type=manifest.command_type, - ) - class CommandReply(CommandBase): reply_id: str = Field(..., description="unique identifier from request") diff --git a/tests/test_models.py b/tests/test_models.py index 1d5d7c4..dddc428 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -28,11 +28,12 @@ def request_id() -> str: @pytest.mark.parametrize("params", PARAMS) def test_command_manifest(params: Optional[List[CommandParameter]]): for command_type in CommnadType: - assert CommandManifest.create( + assert CommandManifest( action="test", description="some test action", command_type=command_type, - params=params, + params=[] if params is None else params, + handler=None, ) @@ -41,40 +42,22 @@ def test_command(request_id: str, params: Optional[List[CommandParameter]]): request_params: Dict[str, Any] = ( {} if params is None else {x.name: None for x in params} ) - command = CommandManifest.create( + manifest = CommandManifest( action="test", description="some test action", command_type=CommnadType.WITHOUT_REPLY, - params=params, + params=[] if params is None else params, + handler=None, ) - assert CommandRequest.from_manifest( - manifest=command, request_id=request_id, params=request_params + assert CommandRequest( + request_id=request_id, + action=manifest.action, + command_type=manifest.command_type, + params=request_params, ) -@pytest.mark.parametrize("params", PARAMS) -def test_params_not_respecting_manifest( - request_id: str, params: Optional[List[CommandParameter]] -): - command = CommandManifest.create( - action="test", - description="some test action", - command_type=CommnadType.WAIT_FOR_REPLY, - params=params, - ) - - if params: - with pytest.raises(ValueError): - assert CommandRequest.from_manifest( - manifest=command, request_id=request_id, params={} - ) - else: - assert CommandRequest.from_manifest( - manifest=command, request_id=request_id, params={} - ) - - @pytest.mark.parametrize("params", PARAMS) def test_msgpack_serialization_deserialization( request_id: str, params: Optional[List[CommandParameter]] @@ -83,15 +66,19 @@ def test_msgpack_serialization_deserialization( request_params: Dict[str, Any] = ( {} if params is None else {x.name: None for x in params} ) - manifest = CommandManifest.create( + manifest = CommandManifest( action="test", description="some test action", command_type=CommnadType.WAIT_FOR_REPLY, - params=params, + params=[] if params is None else params, + handler=None, ) - command_request = CommandRequest.from_manifest( - manifest=manifest, request_id=request_id, params=request_params + command_request = CommandRequest( + request_id=request_id, + action=manifest.action, + command_type=manifest.command_type, + params=request_params, ) assert command_request == CommandRequest.from_bytes(command_request.to_bytes()) From ddcdc4ba4794e68a08514ca0267d31d1cef37f77 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 08:40:37 +0100 Subject: [PATCH 23/86] more models extension --- src/osparc_control/models.py | 32 +++++++++++++++++++++++++++++--- tests/test_models.py | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index bc27aa6..82b62bc 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -1,10 +1,8 @@ -from cgitb import handler from enum import Enum from typing import Any, Callable, Dict, List, Optional -from xml.dom import InvalidAccessErr import umsgpack -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, Extra, Field, validator class CommandBase(BaseModel): @@ -36,9 +34,11 @@ class CommnadType(str, Enum): # no reply is expected for this command # nothing will be awaited WITHOUT_REPLY = "WITHOUT_REPLY" + # a reply is expected and the user must check # for the results WITH_REPLAY = "WITH_REPLAY" + # user requests a parameter that he would like to have # immediately, the request will be blocked until # a value is returned @@ -72,6 +72,32 @@ class CommandRequest(CommandBase): ) +class CommandAccepted(CommandBase): + request_id: str = Field(..., description="unique identifier from request") + accepted: bool = Field( + ..., description="True if command is correctly formatted otherwise False" + ) + error_message: Optional[str] = Field( + None, + description=( + "A mesage displayed to the user in case something went wrong. " + "Will always be present if accepted=False" + ), + ) + + @validator("error_message") + @classmethod + def error_message_present_if_not_accepted( + cls, v: str, values: Dict[str, Any] + ) -> Optional[str]: + if values["accepted"] is False and v is None: + raise ValueError("error_message must not be None when accepted is False") + + if values["accepted"] is True and v is not None: + raise ValueError("error_message must be None when accepted is True") + return v + + class CommandReply(CommandBase): reply_id: str = Field(..., description="unique identifier from request") payload: Any = Field(..., description="user defined value for the command") diff --git a/tests/test_models.py b/tests/test_models.py index dddc428..4731e97 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -3,10 +3,12 @@ import pytest import umsgpack +from pydantic import ValidationError from osparc_control.models import ( CommandManifest, CommandParameter, + CommandAccepted, CommandReply, CommandRequest, CommnadType, @@ -95,3 +97,21 @@ def test_command_reply_payloads_serialization_deserialization( command_reply = CommandReply(reply_id=request_id, payload=payload) assert command_reply assert command_reply == CommandReply.from_bytes(command_reply.to_bytes()) + + +def test_command_accepted_ok(request_id: str): + assert CommandAccepted(request_id=request_id, accepted=True, error_message=None) + assert CommandAccepted( + request_id=request_id, accepted=False, error_message="some error" + ) + + +def test_command_accepted_fails(request_id: str): + with pytest.raises(ValidationError): + assert CommandAccepted( + request_id=request_id, accepted=False, error_message=None + ) + with pytest.raises(ValidationError): + assert CommandAccepted( + request_id=request_id, accepted=True, error_message="some error" + ) From 0333bdf227163c642d67edb624761effbd67bb61 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 08:53:37 +0100 Subject: [PATCH 24/86] adds parameter validation to manifest --- src/osparc_control/models.py | 8 ++++++++ tests/test_models.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 82b62bc..72b08f1 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -1,5 +1,6 @@ from enum import Enum from typing import Any, Callable, Dict, List, Optional +from xml.dom import ValidationErr import umsgpack from pydantic import BaseModel, Extra, Field, validator @@ -62,6 +63,13 @@ class CommandManifest(BaseModel): ), ) + @validator("params") + @classmethod + def ensure_unique_parameter_names(cls, v) -> List[CommandParameter]: + if len(v) != len({x.name for x in v}): + raise ValueError(f"Duplicate CommandParameter name found in {v}") + return v + class CommandRequest(CommandBase): request_id: str = Field(..., description="unique identifier") diff --git a/tests/test_models.py b/tests/test_models.py index 4731e97..182f1bd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -115,3 +115,17 @@ def test_command_accepted_fails(request_id: str): assert CommandAccepted( request_id=request_id, accepted=True, error_message="some error" ) + + +def test_duplicate_command_parameter_name_in_manifest(): + with pytest.raises(ValidationError): + CommandManifest( + action="test", + description="with invalid paramters", + params=[ + CommandParameter(name="a", description="ok"), + CommandParameter(name="a", description="not allowed same name"), + ], + command_type=CommnadType.WITH_REPLAY, + handler=None, + ) From 2b03027b050f2b3dc5da8f46197ceeadc8c833d7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 10:05:06 +0100 Subject: [PATCH 25/86] models extention --- src/osparc_control/models.py | 7 +++++-- tests/test_models.py | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 72b08f1..ecc3693 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -3,7 +3,7 @@ from xml.dom import ValidationErr import umsgpack -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Extra, Field, PrivateAttr, validator class CommandBase(BaseModel): @@ -47,6 +47,9 @@ class CommnadType(str, Enum): class CommandManifest(BaseModel): + # used internally + _remapped_params: Dict[str, CommandParameter] = PrivateAttr() + action: str = Field(..., description="name of the action to be triggered on remote") description: str = Field(..., description="more details about the action") params: List[CommandParameter] = Field( @@ -80,7 +83,7 @@ class CommandRequest(CommandBase): ) -class CommandAccepted(CommandBase): +class CommandReceived(CommandBase): request_id: str = Field(..., description="unique identifier from request") accepted: bool = Field( ..., description="True if command is correctly formatted otherwise False" diff --git a/tests/test_models.py b/tests/test_models.py index 182f1bd..e18a086 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,7 +8,7 @@ from osparc_control.models import ( CommandManifest, CommandParameter, - CommandAccepted, + CommandReceived, CommandReply, CommandRequest, CommnadType, @@ -100,19 +100,19 @@ def test_command_reply_payloads_serialization_deserialization( def test_command_accepted_ok(request_id: str): - assert CommandAccepted(request_id=request_id, accepted=True, error_message=None) - assert CommandAccepted( + assert CommandReceived(request_id=request_id, accepted=True, error_message=None) + assert CommandReceived( request_id=request_id, accepted=False, error_message="some error" ) def test_command_accepted_fails(request_id: str): with pytest.raises(ValidationError): - assert CommandAccepted( + assert CommandReceived( request_id=request_id, accepted=False, error_message=None ) with pytest.raises(ValidationError): - assert CommandAccepted( + assert CommandReceived( request_id=request_id, accepted=True, error_message="some error" ) From 6adc09c8425f30c18024e091621094ebbef28a96 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 10:05:17 +0100 Subject: [PATCH 26/86] base working example --- src/osparc_control/core.py | 235 +++++++++++++++++++++++++++++------ src/osparc_control/errors.py | 32 +++++ 2 files changed, 231 insertions(+), 36 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 4f581f3..3a8f4f0 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -2,7 +2,7 @@ from queue import Queue from threading import Thread from time import sleep -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union, List from uuid import getnode, uuid4 from pydantic import ValidationError @@ -12,8 +12,15 @@ from osparc_control.transport.zeromq import ZeroMQTransport -from .errors import NoReplyException +from .errors import ( + NoReplyException, + CommnadNotAcceptedException, + NoCommandReceivedArrivedException, + NotAllCommandManifestsHaveHandlersException, + SignatureDoesNotMatchException, +) from .models import ( + CommandReceived, CommandManifest, CommandReply, CommandRequest, @@ -23,8 +30,16 @@ ) from .transport.base_transport import SenderReceiverPair + +_MINUTE: float = 60.0 + WAIT_FOR_MESSAGES: float = 0.01 WAIT_BETWEEN_CHECKS: float = 0.1 +# NOTE: this effectively limits the time between when +# the two remote sides can start to communicate +WAIT_FOR_RECEIVED: float = 1 * _MINUTE + +DEFAULT_LISTEN_PORT: int = 7426 def _generate_request_id() -> str: @@ -46,22 +61,45 @@ def _get_sender_receiver_pair( class SomeEntryPoint: def __init__( - self, remote_host: str, remote_port: int = 7426, listen_port: int = 7426 + self, + remote_host: str, + exposed_interface: List[CommandManifest], + remote_port: int = DEFAULT_LISTEN_PORT, + listen_port: int = DEFAULT_LISTEN_PORT, ) -> None: self._sender_receiver_pair: SenderReceiverPair = _get_sender_receiver_pair( remote_host=remote_host, remote_port=remote_port, listen_port=listen_port ) + def _process_manifest(manifest: CommandManifest) -> CommandManifest: + manifest._remapped_params = {x.name: x for x in manifest.params} + return manifest + + # map by action name for + self._exposed_interface: Dict[str, CommandManifest] = { + x.action: _process_manifest(x) for x in exposed_interface + } + if len(self._exposed_interface) != len(exposed_interface): + raise ValueError( + ( + f"Provided exposed_interface={exposed_interface} " + "contains CommandManifest with same action name." + ) + ) + self._request_tracker: RequestsTracker = {} - # TODO: below must be protected by a lock + # NOTE: deque is thread safe only when used with appends and pops self._incoming_request_tracker = deque() self._out_queue: Queue = Queue() + self._incoming_command_queue: Queue = Queue() - # sending and receving threads - self._sender_thread: Thread = Thread(target=self._sender_worker) - self._receiver_thread: Thread = Thread(target=self._receiver_worker) + # sending and receiving threads + self._sender_thread: Thread = Thread(target=self._sender_worker, daemon=True) + self._receiver_thread: Thread = Thread( + target=self._receiver_worker, daemon=True + ) self._continue: bool = True def _sender_worker(self) -> None: @@ -80,6 +118,74 @@ def _sender_worker(self) -> None: self._sender_receiver_pair.sender_cleanup() + def _handle_command_request(self, response: bytes) -> None: + command_request: Optional[CommandRequest] = CommandRequest.from_bytes(response) + assert command_request + + def _refuse_and_return(error_message: str) -> None: + self._out_queue.put( + CommandReceived( + request_id=command_request.request_id, + accepted=False, + error_message=error_message, + ) + ) + return + + # check if command exists + if command_request.action not in self._exposed_interface: + error_message = ( + f"No registered command found for action={command_request.action}. " + f"Supported actions {list(self._exposed_interface.keys())}" + ) + _refuse_and_return(error_message) + + manifest = self._exposed_interface[command_request.action] + + # check command_type matches the one declared in the manifest + if command_request.command_type != manifest.command_type: + error_message = ( + f"Incoming request command_type {command_request.command_type} do not match " + f"manifest's command_type {manifest.command_type}" + ) + _refuse_and_return(error_message) + + # check if provided parametes match manifest + incoming_params_set = set(command_request.params.keys()) + manifest_params_set = set(manifest._remapped_params.keys()) + if incoming_params_set != manifest_params_set: + error_message = ( + f"Incoming request params {command_request.params} do not match " + f"manifest's params {manifest.params}" + ) + _refuse_and_return(error_message) + + # accept command + self._out_queue.put( + CommandReceived( + request_id=command_request.request_id, + accepted=True, + error_message=None, + ) + ) + + self._incoming_request_tracker.append(command_request) + + def _handle_command_reply(self, response: bytes) -> None: + command_reply: Optional[CommandReply] = CommandReply.from_bytes(response) + assert command_reply + + tracked_request: TrackedRequest = self._request_tracker[command_reply.reply_id] + tracked_request.reply = command_reply + + def _handle_command_received(self, response: bytes) -> None: + command_received: Optional[CommandReceived] = CommandReceived.from_bytes( + response + ) + assert command_received + + self._incoming_command_queue.put_nowait(command_received) + def _receiver_worker(self) -> None: self._sender_receiver_pair.receiver_init() @@ -93,22 +199,21 @@ def _receiver_worker(self) -> None: # NOTE: pydantic does not support polymorphism # SEE https://github.com/samuelcolvin/pydantic/issues/503 - # check if is CommandRequest + # case CommandReceived + try: + self._handle_command_received(response) + except ValidationError: + pass + + # case CommandRequest try: - command_request = CommandRequest.from_bytes(response) - assert command_request - self._incoming_request_tracker.append(command_request) + self._handle_command_request(response) except ValidationError: pass - # check if is CommandReply + # case CommandReply try: - command_reply = CommandReply.from_bytes(response) - assert command_reply - tracked_request: TrackedRequest = self._request_tracker[ - command_reply.reply_id - ] - tracked_request.reply = command_reply + self._handle_command_reply(response) except ValidationError: pass @@ -118,26 +223,40 @@ def _receiver_worker(self) -> None: def _enqueue_call( self, - manifest: CommandManifest, + action: str, params: Optional[Dict[str, Any]], expected_command_type: CommnadType, ) -> CommandRequest: """validates and enqueues the call for delivery to remote""" - request = CommandRequest.from_manifest( - manifest=manifest, request_id=_generate_request_id(), params=params + request = CommandRequest( + request_id=_generate_request_id(), + action=action, + params={} if params is None else params, + command_type=expected_command_type, ) - if request.command_type != expected_command_type: - raise RuntimeError( - f"Request {request} was expected to have command_type={expected_command_type}" - ) - self._request_tracker[request.request_id] = TrackedRequest( request=request, reply=None ) self._out_queue.put(request) + # check command_received status + try: + for attempt in Retrying( + stop=stop_after_delay(WAIT_FOR_RECEIVED), + wait=wait_fixed(WAIT_BETWEEN_CHECKS), + ): + with attempt: + command_received = self._incoming_command_queue.get(block=False) + + if not command_received.accepted: + raise CommnadNotAcceptedException( + command_received.error_message + ) + except RetryError: + raise NoCommandReceivedArrivedException() + return request def start_background_sync(self) -> None: @@ -157,19 +276,19 @@ def stop_background_sync(self) -> None: self._receiver_thread.join() def request_and_forget( - self, manifest: CommandManifest, params: Optional[Dict[str, Any]] = None + self, action: str, params: Optional[Dict[str, Any]] = None ) -> None: """No reply will be provided by remote side for this command""" - self._enqueue_call(manifest, params, CommnadType.WITHOUT_REPLY) + self._enqueue_call(action, params, CommnadType.WITHOUT_REPLY) def request_and_check( - self, manifest: CommandManifest, params: Optional[Dict[str, Any]] = None + self, action: str, params: Optional[Dict[str, Any]] = None ) -> str: """ returns a `request_id` to be used with `check_for_reply` to monitor if a reply to the request was returned. """ - request = self._enqueue_call(manifest, params, CommnadType.WITH_REPLAY) + request = self._enqueue_call(action, params, CommnadType.WITH_REPLAY) return request.request_id def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: @@ -204,7 +323,7 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: def request_and_wait( self, - manifest: CommandManifest, + action: str, timeout: float, params: Optional[Dict[str, Any]] = None, ) -> Optional[Any]: @@ -213,7 +332,7 @@ def request_and_wait( A timeout for this function is required. If the timeout is reached `None` will be returned. """ - request = self._enqueue_call(manifest, params, CommnadType.WAIT_FOR_REPLY) + request = self._enqueue_call(action, params, CommnadType.WAIT_FOR_REPLY) try: for attempt in Retrying( @@ -228,13 +347,57 @@ def request_and_wait( except RetryError: return None - def get_incoming_request(self) -> Optional[CommandRequest]: - """will try to fetch an incoming request, returns None if nothing is present""" + def get_incoming_requests(self) -> List[CommandRequest]: + """retruns all accumulated CommandRequests""" + results = deque() + + # fetch all elements empty + # below implementation is thread-safe try: - return self._incoming_request_tracker.pop() + while True: + results.append(self._incoming_request_tracker.pop()) except IndexError: - return None + pass + + return list(results) def reply_to_command(self, request_id: str, payload: Any) -> None: """provide the reply back to a command""" self._out_queue.put(CommandReply(reply_id=request_id, payload=payload)) + + def process_incoming_requests(self, keep_processing: bool = False) -> None: + """ + Requires the user to define handler on all CommandManifest entries + param: `keep_processing` if True will process in an infinite loop without exiting + """ + all_commands_define_a_handler = all( + manifest.handler is not None + for manifest in self._exposed_interface.values() + ) + if not all_commands_define_a_handler: + manifests_without_handler = [ + x for x in self._exposed_interface.values() if x.handler is None + ] + raise NotAllCommandManifestsHaveHandlersException(manifests_without_handler) + + can_continue = True + while can_continue: + for incoming_request in self.get_incoming_requests(): + # fetch manifest + manifest = self._exposed_interface[incoming_request.action] + + # call handler with params and get the result + assert manifest.handler + try: + result = manifest.handler(**incoming_request.params) + except TypeError as e: + raise SignatureDoesNotMatchException( + manifest.action, incoming_request.params + ) from e + + # reply to command + self.reply_to_command( + request_id=incoming_request.request_id, payload=result + ) + + can_continue = keep_processing diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index 5b0b197..256af3e 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -1,6 +1,38 @@ +from typing import Any, Dict, List +from osparc_control.models import CommandManifest + + class BaseControlException(Exception): """inherited by all exceptions in this moduls""" class NoReplyException(BaseControlException): """Used when retrying for a result""" + + +class CommnadNotAcceptedException(BaseControlException): + """Command was not accepted by remote""" + + +class NoCommandReceivedArrivedException(BaseControlException): + """Reply from remote host did not arrive in time""" + + +class NotAllCommandManifestsHaveHandlersException(Exception): + def __init__(self, manifests_without_handler: List[CommandManifest]) -> None: + super().__init__( + ( + "Not all commands define a handler. Please add one to the " + f"following {manifests_without_handler} to use this." + ) + ) + + +class SignatureDoesNotMatchException(Exception): + def __init__(self, action: str, params: Dict[str, Any]) -> None: + super().__init__( + ( + f"Handler signature did not match. Called for action=" + f"'{action}' with params={params}" + ) + ) From 7e8fea2614ebcaeacf79cf609671832af4df761b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:15:39 +0100 Subject: [PATCH 27/86] working version --- src/osparc_control/core.py | 69 ++++++++++-------------------------- src/osparc_control/models.py | 7 ---- tests/test_models.py | 4 --- 3 files changed, 18 insertions(+), 62 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 3a8f4f0..67bb9e7 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -1,5 +1,5 @@ from collections import deque -from queue import Queue +from queue import Queue, Empty from threading import Thread from time import sleep from typing import Any, Dict, Optional, Tuple, Union, List @@ -16,8 +16,6 @@ NoReplyException, CommnadNotAcceptedException, NoCommandReceivedArrivedException, - NotAllCommandManifestsHaveHandlersException, - SignatureDoesNotMatchException, ) from .models import ( CommandReceived, @@ -59,7 +57,7 @@ def _get_sender_receiver_pair( return SenderReceiverPair(sender=sender, receiver=receiver) -class SomeEntryPoint: +class Engine: def __init__( self, remote_host: str, @@ -112,7 +110,6 @@ def _sender_worker(self) -> None: # exit worker break - print("Message to deliver", message) # send message self._sender_receiver_pair.send_bytes(message.to_bytes()) @@ -146,7 +143,8 @@ def _refuse_and_return(error_message: str) -> None: if command_request.command_type != manifest.command_type: error_message = ( f"Incoming request command_type {command_request.command_type} do not match " - f"manifest's command_type {manifest.command_type}" + f"manifest's command_type {manifest.command_type} for command " + f"{command_request.action}" ) _refuse_and_return(error_message) @@ -242,21 +240,23 @@ def _enqueue_call( self._out_queue.put(request) # check command_received status + command_received: Optional[CommandReceived] = None try: for attempt in Retrying( stop=stop_after_delay(WAIT_FOR_RECEIVED), wait=wait_fixed(WAIT_BETWEEN_CHECKS), + retry_error_cls=Empty, ): with attempt: command_received = self._incoming_command_queue.get(block=False) - - if not command_received.accepted: - raise CommnadNotAcceptedException( - command_received.error_message - ) except RetryError: raise NoCommandReceivedArrivedException() + assert command_received + + if not command_received.accepted: + raise CommnadNotAcceptedException(command_received.error_message) + return request def start_background_sync(self) -> None: @@ -305,9 +305,11 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: ) if tracked_request is None: return False, None - # check for the correct type of request - if not tracked_request.request.command_type != CommnadType.WAIT_FOR_REPLY: + if tracked_request.request.command_type not in { + CommnadType.WAIT_FOR_REPLY, + CommnadType.WITH_REPLAY, + }: raise RuntimeError( ( f"Request {tracked_request.request} not expect a " @@ -336,7 +338,9 @@ def request_and_wait( try: for attempt in Retrying( - stop=stop_after_delay(timeout), wait=wait_fixed(WAIT_BETWEEN_CHECKS) + stop=stop_after_delay(timeout), + wait=wait_fixed(WAIT_BETWEEN_CHECKS), + retry_error_cls=NoReplyException, ): with attempt: reply_received, result = self.check_for_reply(request.request_id) @@ -364,40 +368,3 @@ def get_incoming_requests(self) -> List[CommandRequest]: def reply_to_command(self, request_id: str, payload: Any) -> None: """provide the reply back to a command""" self._out_queue.put(CommandReply(reply_id=request_id, payload=payload)) - - def process_incoming_requests(self, keep_processing: bool = False) -> None: - """ - Requires the user to define handler on all CommandManifest entries - param: `keep_processing` if True will process in an infinite loop without exiting - """ - all_commands_define_a_handler = all( - manifest.handler is not None - for manifest in self._exposed_interface.values() - ) - if not all_commands_define_a_handler: - manifests_without_handler = [ - x for x in self._exposed_interface.values() if x.handler is None - ] - raise NotAllCommandManifestsHaveHandlersException(manifests_without_handler) - - can_continue = True - while can_continue: - for incoming_request in self.get_incoming_requests(): - # fetch manifest - manifest = self._exposed_interface[incoming_request.action] - - # call handler with params and get the result - assert manifest.handler - try: - result = manifest.handler(**incoming_request.params) - except TypeError as e: - raise SignatureDoesNotMatchException( - manifest.action, incoming_request.params - ) from e - - # reply to command - self.reply_to_command( - request_id=incoming_request.request_id, payload=result - ) - - can_continue = keep_processing diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index ecc3693..9469163 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -58,13 +58,6 @@ class CommandManifest(BaseModel): command_type: CommnadType = Field( ..., description="describes the command type, behaviour and usage" ) - handler: Optional[Callable] = Field( - None, - description=( - "if the user provides a callable it will be called to handle" - "incoming requests" - ), - ) @validator("params") @classmethod diff --git a/tests/test_models.py b/tests/test_models.py index e18a086..b802006 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -35,7 +35,6 @@ def test_command_manifest(params: Optional[List[CommandParameter]]): description="some test action", command_type=command_type, params=[] if params is None else params, - handler=None, ) @@ -49,7 +48,6 @@ def test_command(request_id: str, params: Optional[List[CommandParameter]]): description="some test action", command_type=CommnadType.WITHOUT_REPLY, params=[] if params is None else params, - handler=None, ) assert CommandRequest( @@ -73,7 +71,6 @@ def test_msgpack_serialization_deserialization( description="some test action", command_type=CommnadType.WAIT_FOR_REPLY, params=[] if params is None else params, - handler=None, ) command_request = CommandRequest( @@ -127,5 +124,4 @@ def test_duplicate_command_parameter_name_in_manifest(): CommandParameter(name="a", description="not allowed same name"), ], command_type=CommnadType.WITH_REPLAY, - handler=None, ) From e7db3e1ef58bffae1d375d8bc9bec6315f7c197d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:16:00 +0100 Subject: [PATCH 28/86] removed unused errors --- src/osparc_control/errors.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index 256af3e..c3b7564 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -16,23 +16,3 @@ class CommnadNotAcceptedException(BaseControlException): class NoCommandReceivedArrivedException(BaseControlException): """Reply from remote host did not arrive in time""" - - -class NotAllCommandManifestsHaveHandlersException(Exception): - def __init__(self, manifests_without_handler: List[CommandManifest]) -> None: - super().__init__( - ( - "Not all commands define a handler. Please add one to the " - f"following {manifests_without_handler} to use this." - ) - ) - - -class SignatureDoesNotMatchException(Exception): - def __init__(self, action: str, params: Dict[str, Any]) -> None: - super().__init__( - ( - f"Handler signature did not match. Called for action=" - f"'{action}' with params={params}" - ) - ) From 712e85d301c58242576b403885bf4998be3eac38 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:22:29 +0100 Subject: [PATCH 29/86] refator renaming --- src/osparc_control/core.py | 14 +++++++------- src/osparc_control/models.py | 20 ++++++++------------ tests/test_models.py | 4 ++-- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 67bb9e7..25f5fe7 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -275,20 +275,20 @@ def stop_background_sync(self) -> None: self._sender_thread.join() self._receiver_thread.join() - def request_and_forget( + def request_without_reply( self, action: str, params: Optional[Dict[str, Any]] = None ) -> None: """No reply will be provided by remote side for this command""" self._enqueue_call(action, params, CommnadType.WITHOUT_REPLY) - def request_and_check( + def request_with_delayed_reply( self, action: str, params: Optional[Dict[str, Any]] = None ) -> str: """ returns a `request_id` to be used with `check_for_reply` to monitor if a reply to the request was returned. """ - request = self._enqueue_call(action, params, CommnadType.WITH_REPLAY) + request = self._enqueue_call(action, params, CommnadType.WITH_DELAYED_REPLY) return request.request_id def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: @@ -307,8 +307,8 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: return False, None # check for the correct type of request if tracked_request.request.command_type not in { - CommnadType.WAIT_FOR_REPLY, - CommnadType.WITH_REPLAY, + CommnadType.WITH_IMMEDIATE_REPLY, + CommnadType.WITH_DELAYED_REPLY, }: raise RuntimeError( ( @@ -323,7 +323,7 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: return True, tracked_request.reply.payload - def request_and_wait( + def request_with_immediate_reply( self, action: str, timeout: float, @@ -334,7 +334,7 @@ def request_and_wait( A timeout for this function is required. If the timeout is reached `None` will be returned. """ - request = self._enqueue_call(action, params, CommnadType.WAIT_FOR_REPLY) + request = self._enqueue_call(action, params, CommnadType.WITH_IMMEDIATE_REPLY) try: for attempt in Retrying( diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 9469163..ed89047 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -29,21 +29,17 @@ class CommandParameter(BaseModel): class CommnadType(str, Enum): - # NOTE: ANE -> KZ: can you please let me know if names and descriptions make sense? - # please suggest better ones - - # no reply is expected for this command - # nothing will be awaited + # the command expects no reply WITHOUT_REPLY = "WITHOUT_REPLY" - # a reply is expected and the user must check - # for the results - WITH_REPLAY = "WITH_REPLAY" + # the command will provide a reply + # the suer is require to check for the results + # of this reply + WITH_DELAYED_REPLY = "WITH_DELAYED_REPLY" - # user requests a parameter that he would like to have - # immediately, the request will be blocked until - # a value is returned - WAIT_FOR_REPLY = "WAIT_FOR_REPLY" + # the command will return the result immediately + # and user code will be blocked until reply arrives + WITH_IMMEDIATE_REPLY = "WITH_IMMEDIATE_REPLY" class CommandManifest(BaseModel): diff --git a/tests/test_models.py b/tests/test_models.py index b802006..98283f1 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -69,7 +69,7 @@ def test_msgpack_serialization_deserialization( manifest = CommandManifest( action="test", description="some test action", - command_type=CommnadType.WAIT_FOR_REPLY, + command_type=CommnadType.WITH_IMMEDIATE_REPLY, params=[] if params is None else params, ) @@ -123,5 +123,5 @@ def test_duplicate_command_parameter_name_in_manifest(): CommandParameter(name="a", description="ok"), CommandParameter(name="a", description="not allowed same name"), ], - command_type=CommnadType.WITH_REPLAY, + command_type=CommnadType.WITH_DELAYED_REPLY, ) From b12239c5f015bfd8eb460e7e49b905590982cf1a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:40:51 +0100 Subject: [PATCH 30/86] renamed methods --- src/osparc_control/__init__.py | 2 ++ src/osparc_control/__main__.py | 12 ------------ src/osparc_control/core.py | 2 +- tests/test_main.py | 17 ----------------- 4 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 src/osparc_control/__main__.py delete mode 100644 tests/test_main.py diff --git a/src/osparc_control/__init__.py b/src/osparc_control/__init__.py index a6e7db7..442636b 100644 --- a/src/osparc_control/__init__.py +++ b/src/osparc_control/__init__.py @@ -1 +1,3 @@ """Osparc Control.""" +from .models import CommandManifest, CommandParameter, CommnadType +from .core import ControlInterface diff --git a/src/osparc_control/__main__.py b/src/osparc_control/__main__.py deleted file mode 100644 index 4523041..0000000 --- a/src/osparc_control/__main__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Command-line interface.""" -import click - - -@click.command() -@click.version_option() -def main() -> None: - """Osparc Control.""" - - -if __name__ == "__main__": - main(prog_name="osparc-control") # pragma: no cover diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 25f5fe7..bf7ef49 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -57,7 +57,7 @@ def _get_sender_receiver_pair( return SenderReceiverPair(sender=sender, receiver=receiver) -class Engine: +class ControlInterface: def __init__( self, remote_host: str, diff --git a/tests/test_main.py b/tests/test_main.py deleted file mode 100644 index 7d3f3f2..0000000 --- a/tests/test_main.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Test cases for the __main__ module.""" -import pytest -from click.testing import CliRunner - -from osparc_control import __main__ - - -@pytest.fixture -def runner() -> CliRunner: - """Fixture for invoking command-line interfaces.""" - return CliRunner() - - -def test_main_succeeds(runner: CliRunner) -> None: - """It exits with a status code of zero.""" - result = runner.invoke(__main__.main) - assert result.exit_code == 0 From 04a7dd3dd6fd07357ef6b101df426ed3ca07324d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:43:52 +0100 Subject: [PATCH 31/86] add simple example --- examples/base_time_add/README.md | 14 ++++ examples/base_time_add/sidecar_controller.py | 34 +++++++++ examples/base_time_add/sidecar_solver.py | 38 ++++++++++ examples/base_time_add/time_solver.py | 74 ++++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 examples/base_time_add/README.md create mode 100644 examples/base_time_add/sidecar_controller.py create mode 100644 examples/base_time_add/sidecar_solver.py create mode 100644 examples/base_time_add/time_solver.py diff --git a/examples/base_time_add/README.md b/examples/base_time_add/README.md new file mode 100644 index 0000000..e4bc512 --- /dev/null +++ b/examples/base_time_add/README.md @@ -0,0 +1,14 @@ +# About + +This example consists of a `time_solver`. Which can add, the current time by a provided value. + + +# Usage + +In one terminal run `sidecar_controller.py`. +In a second terminal run `time_solver.py`. It will load data from the `sidecar_solver.py` to use when communicating with the `sidecar_controller.py` + + +# In this example + +Only the `solver` exposes an interface that can be queried. The `controller` does not have an exposed interface. diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py new file mode 100644 index 0000000..d631f96 --- /dev/null +++ b/examples/base_time_add/sidecar_controller.py @@ -0,0 +1,34 @@ +from osparc_control import ControlInterface + + +control_interface = ControlInterface( + remote_host="localhost", exposed_interface=[], remote_port=1234, listen_port=1235 +) +control_interface.start_background_sync() + +# add_internal_time + +add_params = {"a": 10} +print("Will add ", add_params) +request_id = control_interface.request_with_delayed_reply( + "add_internal_time", params=add_params +) + +has_result = False +result = None +while not has_result: + has_result, result = control_interface.check_for_reply(request_id=request_id) + +print("result of addition", result) + +# get_time + +print("getting solver time") +solver_time = control_interface.request_with_immediate_reply("get_time", timeout=4.0) +print("solver time", solver_time) + +print("sending command to print internal status") +control_interface.request_without_reply("print_status") + + +control_interface.stop_background_sync() diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py new file mode 100644 index 0000000..660dafb --- /dev/null +++ b/examples/base_time_add/sidecar_solver.py @@ -0,0 +1,38 @@ +from osparc_control import ( + ControlInterface, + CommandManifest, + CommandParameter, + CommnadType, +) + + +command_add = CommandManifest( + action="add_internal_time", + description="adds internal time to a provided paramter", + params=[ + CommandParameter(name="a", description="param to add to internal time"), + ], + command_type=CommnadType.WITH_DELAYED_REPLY, +) + +command_get_time = CommandManifest( + action="get_time", + description="gets the time", + params=[], + command_type=CommnadType.WITH_IMMEDIATE_REPLY, +) + +command_print_solver_status = CommandManifest( + action="print_status", + description="prints the status of the solver", + params=[], + command_type=CommnadType.WITHOUT_REPLY, +) + + +control_interface = ControlInterface( + remote_host="localhost", + exposed_interface=[command_add, command_get_time, command_print_solver_status], + remote_port=1235, + listen_port=1234, +) diff --git a/examples/base_time_add/time_solver.py b/examples/base_time_add/time_solver.py new file mode 100644 index 0000000..ba40e8b --- /dev/null +++ b/examples/base_time_add/time_solver.py @@ -0,0 +1,74 @@ +from os import access +from time import sleep +from osparc_control.core import ControlInterface +from osparc_control.models import CommandRequest + +from sidecar_solver import ( + control_interface, + command_add, + command_get_time, + command_print_solver_status, +) + + +def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: + if request.action == command_add.action: + sum_result = time_solver._add(**request.params) + time_solver.control_interface.reply_to_command( + request_id=request.request_id, payload=sum_result + ) + return + + if request.action == command_get_time.action: + time_solver.control_interface.reply_to_command( + request_id=request.request_id, payload=time_solver.time + ) + return + + if request.action == command_print_solver_status.action: + print("Solver internal status", time_solver) + # finally exit + time_solver.can_continue = False + return + + +class TimeSolver: + def __init__( + self, initial_time: float, control_interface: ControlInterface + ) -> None: + self.time = initial_time + self.control_interface = control_interface + + # internal time tick + self.sleep_internal: float = 0.1 + self.can_continue: bool = True + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} time={self.time}, sleep_interval={self.sleep_internal}>" + + def _add(self, a: float) -> float: + return self.time + a + + def run(self) -> None: + """main loop of the solver""" + while self.can_continue: + # process incoming requests from remote + for request in self.control_interface.get_incoming_requests(): + handle_inputs(time_solver=self, request=request) + + # process internal stuff + self.time += 1 + sleep(self.sleep_internal) + + +def main() -> None: + control_interface.start_background_sync() + + solver = TimeSolver(initial_time=0, control_interface=control_interface) + solver.run() + + control_interface.stop_background_sync() + + +if __name__ == "__main__": + main() From 1602e549b2854f7c9450b4dcdca7a4d64d8eab10 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:44:42 +0100 Subject: [PATCH 32/86] end of line --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f696d45..337f9ef 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ /docs/_build/ /src/*.egg-info/ __pycache__/ -*ignore* \ No newline at end of file +*ignore* From 677f86653c90b75b6d24e1d38cab821640a4289f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 11:48:51 +0100 Subject: [PATCH 33/86] updated codestyle --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 40c6159..464af82 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # poetry is required on your system # suggested installation method -# or refer to official docs +# or refer to official docs # https://python-poetry.org/docs/ .PHONY: install-poetry install-poetry: @@ -19,7 +19,7 @@ tests: # run tests on lowest python interpreter nox -r -s tests -p 3.6 .PHONY: nox-36 -nox-36: # runs nox with python 3.6 +nox-36: # runs nox with python 3.6 nox -p 3.6 .PHONY: tests-dev @@ -30,3 +30,9 @@ tests-dev: docs: # runs and displays docs #runs with py3.6 change the noxfile.py to use different interpreter version nox -r -s docs + + +.PHONY: codestyle +codestyle: # runs codestyle enforcement + isort . + black . From 46631290732d2fc90de040dc8b4560c38b192230 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 12:10:30 +0100 Subject: [PATCH 34/86] pre-commit checks enabled --- .flake8 | 2 +- examples/base_time_add/README.md | 2 - examples/base_time_add/sidecar_solver.py | 10 +-- examples/base_time_add/time_solver.py | 19 ++--- noxfile.py | 2 +- src/osparc_control/__init__.py | 6 +- src/osparc_control/core.py | 77 +++++++++---------- src/osparc_control/errors.py | 14 ++-- src/osparc_control/models.py | 16 +++- .../transport/base_transport.py | 15 ++-- src/osparc_control/transport/in_memory.py | 6 +- src/osparc_control/transport/zeromq.py | 15 ++-- tests/test_models.py | 19 ++--- tests/test_transport.py | 8 +- 14 files changed, 111 insertions(+), 100 deletions(-) diff --git a/.flake8 b/.flake8 index 0f46431..75f1af5 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -select = B,B9,C,D,DAR,E,F,N,RST,S,W +select = B,B9,C,DAR,E,F,N,RST,S,W ignore = E203,E501,RST201,RST203,RST301,W503 max-line-length = 80 max-complexity = 10 diff --git a/examples/base_time_add/README.md b/examples/base_time_add/README.md index e4bc512..3aabad0 100644 --- a/examples/base_time_add/README.md +++ b/examples/base_time_add/README.md @@ -2,13 +2,11 @@ This example consists of a `time_solver`. Which can add, the current time by a provided value. - # Usage In one terminal run `sidecar_controller.py`. In a second terminal run `time_solver.py`. It will load data from the `sidecar_solver.py` to use when communicating with the `sidecar_controller.py` - # In this example Only the `solver` exposes an interface that can be queried. The `controller` does not have an exposed interface. diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py index 660dafb..af0676f 100644 --- a/examples/base_time_add/sidecar_solver.py +++ b/examples/base_time_add/sidecar_solver.py @@ -1,9 +1,7 @@ -from osparc_control import ( - ControlInterface, - CommandManifest, - CommandParameter, - CommnadType, -) +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommnadType +from osparc_control import ControlInterface command_add = CommandManifest( diff --git a/examples/base_time_add/time_solver.py b/examples/base_time_add/time_solver.py index ba40e8b..d341fcc 100644 --- a/examples/base_time_add/time_solver.py +++ b/examples/base_time_add/time_solver.py @@ -1,15 +1,13 @@ -from os import access from time import sleep + +from sidecar_solver import command_add +from sidecar_solver import command_get_time +from sidecar_solver import command_print_solver_status +from sidecar_solver import control_interface + from osparc_control.core import ControlInterface from osparc_control.models import CommandRequest -from sidecar_solver import ( - control_interface, - command_add, - command_get_time, - command_print_solver_status, -) - def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: if request.action == command_add.action: @@ -44,7 +42,10 @@ def __init__( self.can_continue: bool = True def __repr__(self) -> str: - return f"<{self.__class__.__name__} time={self.time}, sleep_interval={self.sleep_internal}>" + return ( + f"<{self.__class__.__name__} time={self.time}, " + f"sleep_interval={self.sleep_internal}>" + ) def _add(self, a: float) -> float: return self.time + a diff --git a/noxfile.py b/noxfile.py index ed02bc1..d1c1eb0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -195,7 +195,7 @@ def docs(session: Session) -> None: """Build and serve the documentation with live reloading on file changes.""" args = session.posargs or [ "--host", - "0.0.0.0", + "0.0.0.0", # noqa: S104 "--open-browser", "docs", "docs/_build", diff --git a/src/osparc_control/__init__.py b/src/osparc_control/__init__.py index 442636b..b80c2ed 100644 --- a/src/osparc_control/__init__.py +++ b/src/osparc_control/__init__.py @@ -1,3 +1,7 @@ """Osparc Control.""" -from .models import CommandManifest, CommandParameter, CommnadType from .core import ControlInterface +from .models import CommandManifest +from .models import CommandParameter +from .models import CommnadType + +__all__ = ["ControlInterface", "CommandManifest", "CommandParameter", "CommnadType"] diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index bf7ef49..4e91943 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -1,32 +1,35 @@ from collections import deque -from queue import Queue, Empty +from queue import Empty +from queue import Queue from threading import Thread from time import sleep -from typing import Any, Dict, Optional, Tuple, Union, List -from uuid import getnode, uuid4 +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union +from uuid import getnode +from uuid import uuid4 from pydantic import ValidationError -from tenacity import RetryError, Retrying +from tenacity import RetryError +from tenacity import Retrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from osparc_control.transport.zeromq import ZeroMQTransport - -from .errors import ( - NoReplyException, - CommnadNotAcceptedException, - NoCommandReceivedArrivedException, -) -from .models import ( - CommandReceived, - CommandManifest, - CommandReply, - CommandRequest, - CommnadType, - RequestsTracker, - TrackedRequest, -) +from .errors import CommnadNotAcceptedError +from .errors import NoCommandReceivedArrivedError +from .errors import NoReplyError +from .models import CommandManifest +from .models import CommandReceived +from .models import CommandReply +from .models import CommandRequest +from .models import CommnadType +from .models import RequestsTracker +from .models import TrackedRequest from .transport.base_transport import SenderReceiverPair +from osparc_control.transport.zeromq import ZeroMQTransport _MINUTE: float = 60.0 @@ -80,10 +83,8 @@ def _process_manifest(manifest: CommandManifest) -> CommandManifest: } if len(self._exposed_interface) != len(exposed_interface): raise ValueError( - ( - f"Provided exposed_interface={exposed_interface} " - "contains CommandManifest with same action name." - ) + f"Provided exposed_interface={exposed_interface} " + "contains CommandManifest with same action name." ) self._request_tracker: RequestsTracker = {} @@ -117,7 +118,7 @@ def _sender_worker(self) -> None: def _handle_command_request(self, response: bytes) -> None: command_request: Optional[CommandRequest] = CommandRequest.from_bytes(response) - assert command_request + assert command_request # noqa: S101 def _refuse_and_return(error_message: str) -> None: self._out_queue.put( @@ -142,9 +143,9 @@ def _refuse_and_return(error_message: str) -> None: # check command_type matches the one declared in the manifest if command_request.command_type != manifest.command_type: error_message = ( - f"Incoming request command_type {command_request.command_type} do not match " - f"manifest's command_type {manifest.command_type} for command " - f"{command_request.action}" + f"Incoming request command_type {command_request.command_type} " + f"do not match manifest's command_type {manifest.command_type} " + f"for command {command_request.action}" ) _refuse_and_return(error_message) @@ -171,7 +172,7 @@ def _refuse_and_return(error_message: str) -> None: def _handle_command_reply(self, response: bytes) -> None: command_reply: Optional[CommandReply] = CommandReply.from_bytes(response) - assert command_reply + assert command_reply # noqa: S101 tracked_request: TrackedRequest = self._request_tracker[command_reply.reply_id] tracked_request.reply = command_reply @@ -180,7 +181,7 @@ def _handle_command_received(self, response: bytes) -> None: command_received: Optional[CommandReceived] = CommandReceived.from_bytes( response ) - assert command_received + assert command_received # noqa: S101 self._incoming_command_queue.put_nowait(command_received) @@ -250,12 +251,12 @@ def _enqueue_call( with attempt: command_received = self._incoming_command_queue.get(block=False) except RetryError: - raise NoCommandReceivedArrivedException() + raise NoCommandReceivedArrivedError() from None - assert command_received + assert command_received # noqa: S101 if not command_received.accepted: - raise CommnadNotAcceptedException(command_received.error_message) + raise CommnadNotAcceptedError(command_received.error_message) return request @@ -311,10 +312,8 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: CommnadType.WITH_DELAYED_REPLY, }: raise RuntimeError( - ( - f"Request {tracked_request.request} not expect a " - f"reply, found reply {tracked_request.reply}" - ) + f"Request {tracked_request.request} not expect a " + f"reply, found reply {tracked_request.reply}" ) # check if reply was received @@ -340,12 +339,12 @@ def request_with_immediate_reply( for attempt in Retrying( stop=stop_after_delay(timeout), wait=wait_fixed(WAIT_BETWEEN_CHECKS), - retry_error_cls=NoReplyException, + retry_error_cls=NoReplyError, ): with attempt: reply_received, result = self.check_for_reply(request.request_id) if not reply_received: - raise NoReplyException() + raise NoReplyError() return result except RetryError: diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index c3b7564..3f18738 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -1,18 +1,14 @@ -from typing import Any, Dict, List -from osparc_control.models import CommandManifest +class BaseControlError(Exception): + """inherited by all exceptions in this module""" -class BaseControlException(Exception): - """inherited by all exceptions in this moduls""" - - -class NoReplyException(BaseControlException): +class NoReplyError(BaseControlError): """Used when retrying for a result""" -class CommnadNotAcceptedException(BaseControlException): +class CommnadNotAcceptedError(BaseControlError): """Command was not accepted by remote""" -class NoCommandReceivedArrivedException(BaseControlException): +class NoCommandReceivedArrivedError(BaseControlError): """Reply from remote host did not arrive in time""" diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index ed89047..298f07b 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -1,9 +1,15 @@ from enum import Enum -from typing import Any, Callable, Dict, List, Optional -from xml.dom import ValidationErr +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import umsgpack -from pydantic import BaseModel, Extra, Field, PrivateAttr, validator +from pydantic import BaseModel +from pydantic import Extra +from pydantic import Field +from pydantic import PrivateAttr +from pydantic import validator class CommandBase(BaseModel): @@ -24,7 +30,9 @@ class CommandParameter(BaseModel): ) description: str = Field( ..., - description="provide more information to the user, how this command should be used", + description=( + "provide more information to the user, how this command should be used" + ), ) diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index 407fba8..0af8942 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -1,14 +1,15 @@ -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod from typing import Optional class BaseTransport(ABCMeta): @abstractmethod - def send_bytes(self, payload: bytes) -> None: + def send_bytes(self, payload: bytes) -> None: # noqa: N804 """sends bytes to remote""" @abstractmethod - def receive_bytes(self) -> Optional[bytes]: + def receive_bytes(self) -> Optional[bytes]: # noqa: N804 """ returns bytes from remote NOTE: this must never wait, it returns None if @@ -16,25 +17,25 @@ def receive_bytes(self) -> Optional[bytes]: """ @abstractmethod - def sender_init(self) -> None: + def sender_init(self) -> None: # noqa: N804 """ Some libraries require thread specific context. This will be called by the thread once its started """ @abstractmethod - def receiver_init(self) -> None: + def receiver_init(self) -> None: # noqa: N804 """ Some libraries require thread specific context. This will be called by the thread once its started """ - def sender_cleanup(self) -> None: + def sender_cleanup(self) -> None: # noqa: N804 """ Some libraries require cleanup when done with them """ - def receiver_cleanup(self) -> None: + def receiver_cleanup(self) -> None: # noqa: N804 """ Some libraries require cleanup when done with them """ diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py index 54c898d..b08ceff 100644 --- a/src/osparc_control/transport/in_memory.py +++ b/src/osparc_control/transport/in_memory.py @@ -1,5 +1,7 @@ -from queue import Empty, Queue -from typing import Dict, Optional +from queue import Empty +from queue import Queue +from typing import Dict +from typing import Optional from .base_transport import BaseTransport diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py index ebb9f57..d363d68 100644 --- a/src/osparc_control/transport/zeromq.py +++ b/src/osparc_control/transport/zeromq.py @@ -1,7 +1,8 @@ from typing import Optional import zmq -from tenacity import RetryError, Retrying +from tenacity import RetryError +from tenacity import Retrying from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed @@ -20,14 +21,14 @@ def __init__(self, listen_port: int, remote_host: str, remote_port: int): self._recv_contex: Optional[zmq.context.Context] = None def send_bytes(self, payload: bytes) -> None: - assert self._send_socket + assert self._send_socket # noqa: S101 self._send_socket.send(payload) def receive_bytes( self, retry_count: int = 3, wait_between: float = 0.01 ) -> Optional[bytes]: - assert self._recv_socket + assert self._recv_socket # noqa: S101 # try to fetch a message, usning unlocking sockets does not guarantee # that data is always present, retry 3 times in a short amount of time @@ -53,13 +54,13 @@ def receiver_init(self) -> None: self._recv_socket.connect(f"tcp://{self.remote_host}:{self.remote_port}") def sender_cleanup(self) -> None: - assert self._send_socket + assert self._send_socket # noqa: S101 self._send_socket.close() - assert self._send_contex + assert self._send_contex # noqa: S101 self._send_contex.term() def receiver_cleanup(self) -> None: - assert self._recv_socket + assert self._recv_socket # noqa: S101 self._recv_socket.close() - assert self._recv_contex + assert self._recv_contex # noqa: S101 self._recv_contex.term() diff --git a/tests/test_models.py b/tests/test_models.py index 98283f1..d26d253 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,18 +1,19 @@ import json -from typing import Any, Dict, List, Optional +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import pytest import umsgpack from pydantic import ValidationError -from osparc_control.models import ( - CommandManifest, - CommandParameter, - CommandReceived, - CommandReply, - CommandRequest, - CommnadType, -) +from osparc_control.models import CommandManifest +from osparc_control.models import CommandParameter +from osparc_control.models import CommandReceived +from osparc_control.models import CommandReply +from osparc_control.models import CommandRequest +from osparc_control.models import CommnadType @pytest.fixture diff --git a/tests/test_transport.py b/tests/test_transport.py index 52dd47a..459b2c6 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,8 +1,10 @@ -from typing import Iterable, Type +from typing import Iterable +from typing import Type import pytest -from osparc_control.transport.base_transport import BaseTransport, SenderReceiverPair +from osparc_control.transport.base_transport import BaseTransport +from osparc_control.transport.base_transport import SenderReceiverPair from osparc_control.transport.in_memory import InMemoryTransport from osparc_control.transport.zeromq import ZeroMQTransport @@ -62,7 +64,7 @@ def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair): def test_receive_nothing(sender_receiver_pair: SenderReceiverPair): - assert sender_receiver_pair.receive_bytes() == None + assert sender_receiver_pair.receive_bytes() is None def test_receive_returns_none_if_no_message_available(): From 3d4760a2571b3515a36e50349d86fe9db113cf5c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:36:16 +0100 Subject: [PATCH 35/86] fixed issues with mypy --- Makefile | 4 ++ examples/base_time_add/sidecar_controller.py | 2 +- src/osparc_control/core.py | 33 ++++++++------ src/osparc_control/models.py | 8 ++-- .../transport/base_transport.py | 6 ++- src/osparc_control/transport/in_memory.py | 4 +- src/osparc_control/transport/zeromq.py | 44 +++++++++++-------- tests/test_models.py | 16 +++---- tests/test_transport.py | 26 ++++++----- 9 files changed, 84 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 464af82..20c5581 100644 --- a/Makefile +++ b/Makefile @@ -36,3 +36,7 @@ docs: # runs and displays docs codestyle: # runs codestyle enforcement isort . black . + +.PHONY: mypy +mypy: # runs mypy + mypy src tests docs/conf.py diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py index d631f96..bddfd94 100644 --- a/examples/base_time_add/sidecar_controller.py +++ b/examples/base_time_add/sidecar_controller.py @@ -24,7 +24,7 @@ # get_time print("getting solver time") -solver_time = control_interface.request_with_immediate_reply("get_time", timeout=4.0) +solver_time = control_interface.request_with_immediate_reply("get_time", timeout=1.0) print("solver time", solver_time) print("sending command to print internal status") diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 4e91943..e7e7bab 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -4,6 +4,7 @@ from threading import Thread from time import sleep from typing import Any +from typing import Deque from typing import Dict from typing import List from typing import Optional @@ -89,10 +90,12 @@ def _process_manifest(manifest: CommandManifest) -> CommandManifest: self._request_tracker: RequestsTracker = {} # NOTE: deque is thread safe only when used with appends and pops - self._incoming_request_tracker = deque() + self._incoming_request_tracker: Deque[CommandRequest] = deque() - self._out_queue: Queue = Queue() - self._incoming_command_queue: Queue = Queue() + self._out_queue: Queue[ + Optional[Union[CommandRequest, CommandReceived, CommandReply]] + ] = Queue() + self._incoming_command_queue: Queue[Optional[CommandReceived]] = Queue() # sending and receiving threads self._sender_thread: Thread = Thread(target=self._sender_worker, daemon=True) @@ -105,8 +108,9 @@ def _sender_worker(self) -> None: self._sender_receiver_pair.sender_init() while self._continue: - message = self._out_queue.get() - message: Optional[Union[CommandRequest, CommandReply]] = message + message: Optional[ + Union[CommandRequest, CommandReceived, CommandReply] + ] = self._out_queue.get() if message is None: # exit worker break @@ -118,12 +122,13 @@ def _sender_worker(self) -> None: def _handle_command_request(self, response: bytes) -> None: command_request: Optional[CommandRequest] = CommandRequest.from_bytes(response) - assert command_request # noqa: S101 + if command_request is None: + return def _refuse_and_return(error_message: str) -> None: self._out_queue.put( CommandReceived( - request_id=command_request.request_id, + request_id=command_request.request_id, # type: ignore accepted=False, error_message=error_message, ) @@ -246,7 +251,7 @@ def _enqueue_call( for attempt in Retrying( stop=stop_after_delay(WAIT_FOR_RECEIVED), wait=wait_fixed(WAIT_BETWEEN_CHECKS), - retry_error_cls=Empty, + retry_error_cls=Empty, # type: ignore ): with attempt: command_received = self._incoming_command_queue.get(block=False) @@ -335,24 +340,26 @@ def request_with_immediate_reply( """ request = self._enqueue_call(action, params, CommnadType.WITH_IMMEDIATE_REPLY) + result: Optional[Any] = None + try: for attempt in Retrying( stop=stop_after_delay(timeout), wait=wait_fixed(WAIT_BETWEEN_CHECKS), - retry_error_cls=NoReplyError, + retry_error_cls=NoReplyError, # type: ignore ): with attempt: reply_received, result = self.check_for_reply(request.request_id) if not reply_received: raise NoReplyError() - - return result except RetryError: - return None + pass + + return result def get_incoming_requests(self) -> List[CommandRequest]: """retruns all accumulated CommandRequests""" - results = deque() + results: Deque[CommandRequest] = deque() # fetch all elements empty # below implementation is thread-safe diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 298f07b..a376450 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -4,7 +4,7 @@ from typing import List from typing import Optional -import umsgpack +import umsgpack # type: ignore from pydantic import BaseModel from pydantic import Extra from pydantic import Field @@ -14,7 +14,7 @@ class CommandBase(BaseModel): def to_bytes(self) -> bytes: - return umsgpack.packb(self.dict()) + return umsgpack.packb(self.dict()) # type: ignore @classmethod def from_bytes(cls, raw: bytes) -> Optional[Any]: @@ -65,7 +65,9 @@ class CommandManifest(BaseModel): @validator("params") @classmethod - def ensure_unique_parameter_names(cls, v) -> List[CommandParameter]: + def ensure_unique_parameter_names( + cls, v: List[CommandParameter] + ) -> List[CommandParameter]: if len(v) != len({x.name for x in v}): raise ValueError(f"Duplicate CommandParameter name found in {v}") return v diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index 0af8942..ae07346 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -3,7 +3,11 @@ from typing import Optional -class BaseTransport(ABCMeta): +class BaseTransportMeta(ABCMeta): + pass + + +class BaseTransport(metaclass=BaseTransportMeta): @abstractmethod def send_bytes(self, payload: bytes) -> None: # noqa: N804 """sends bytes to remote""" diff --git a/src/osparc_control/transport/in_memory.py b/src/osparc_control/transport/in_memory.py index b08ceff..9680d52 100644 --- a/src/osparc_control/transport/in_memory.py +++ b/src/osparc_control/transport/in_memory.py @@ -6,7 +6,7 @@ from .base_transport import BaseTransport -class InMemoryTransport(metaclass=BaseTransport): +class InMemoryTransport(BaseTransport): """ Non blocking in memory implementation, working with queues. Can only be mixed with threading. @@ -15,7 +15,7 @@ class InMemoryTransport(metaclass=BaseTransport): - fetches data from `source` """ - _SHARED_QUEUES: Dict[str, Queue] = {} + _SHARED_QUEUES: Dict[str, "Queue[bytes]"] = {} def __init__(self, source: str, destination: str): self.source: str = source diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py index d363d68..d20d888 100644 --- a/src/osparc_control/transport/zeromq.py +++ b/src/osparc_control/transport/zeromq.py @@ -5,25 +5,27 @@ from tenacity import Retrying from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed +from zmq import Context +from zmq import Socket from .base_transport import BaseTransport -class ZeroMQTransport(metaclass=BaseTransport): +class ZeroMQTransport(BaseTransport): def __init__(self, listen_port: int, remote_host: str, remote_port: int): self.listen_port: int = listen_port self.remote_host: str = remote_host self.remote_port: int = remote_port - self._recv_socket: Optional[zmq.socket.Socket] = None - self._send_socket: Optional[zmq.socket.Socket] = None - self._send_contex: Optional[zmq.context.Context] = None - self._recv_contex: Optional[zmq.context.Context] = None + self._recv_socket: Optional[Socket] = None + self._send_socket: Optional[Socket] = None + self._send_contex: Optional[Context] = None + self._recv_contex: Optional[Context] = None def send_bytes(self, payload: bytes) -> None: assert self._send_socket # noqa: S101 - self._send_socket.send(payload) + self._send_socket.send(payload) # type: ignore def receive_bytes( self, retry_count: int = 3, wait_between: float = 0.01 @@ -33,34 +35,38 @@ def receive_bytes( # try to fetch a message, usning unlocking sockets does not guarantee # that data is always present, retry 3 times in a short amount of time # this will guarantee the message arrives + message: Optional[bytes] = None try: for attempt in Retrying( stop=stop_after_attempt(retry_count), wait=wait_fixed(wait_between) ): with attempt: - message: bytes = self._recv_socket.recv(zmq.NOBLOCK) - return message + message = self._recv_socket.recv(zmq.NOBLOCK) # type: ignore except RetryError: - return None + pass + + return message def sender_init(self) -> None: - self._send_contex = zmq.Context() - self._send_socket = self._send_contex.socket(zmq.PUSH) - self._send_socket.bind(f"tcp://*:{self.listen_port}") + self._send_contex = zmq.Context() # type: ignore + self._send_socket = self._send_contex.socket(zmq.PUSH) # type: ignore + self._send_socket.bind(f"tcp://*:{self.listen_port}") # type: ignore def receiver_init(self) -> None: - self._recv_contex = zmq.Context() - self._recv_socket = self._recv_contex.socket(zmq.PULL) - self._recv_socket.connect(f"tcp://{self.remote_host}:{self.remote_port}") + self._recv_contex = zmq.Context() # type: ignore + self._recv_socket = self._recv_contex.socket(zmq.PULL) # type: ignore + self._recv_socket.connect( # type: ignore + f"tcp://{self.remote_host}:{self.remote_port}" + ) def sender_cleanup(self) -> None: assert self._send_socket # noqa: S101 - self._send_socket.close() + self._send_socket.close() # type: ignore assert self._send_contex # noqa: S101 - self._send_contex.term() + self._send_contex.term() # type: ignore def receiver_cleanup(self) -> None: assert self._recv_socket # noqa: S101 - self._recv_socket.close() + self._recv_socket.close() # type: ignore assert self._recv_contex # noqa: S101 - self._recv_contex.term() + self._recv_contex.term() # type: ignore diff --git a/tests/test_models.py b/tests/test_models.py index d26d253..e2cd372 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -5,7 +5,7 @@ from typing import Optional import pytest -import umsgpack +import umsgpack # type: ignore from pydantic import ValidationError from osparc_control.models import CommandManifest @@ -29,7 +29,7 @@ def request_id() -> str: @pytest.mark.parametrize("params", PARAMS) -def test_command_manifest(params: Optional[List[CommandParameter]]): +def test_command_manifest(params: Optional[List[CommandParameter]]) -> None: for command_type in CommnadType: assert CommandManifest( action="test", @@ -40,7 +40,7 @@ def test_command_manifest(params: Optional[List[CommandParameter]]): @pytest.mark.parametrize("params", PARAMS) -def test_command(request_id: str, params: Optional[List[CommandParameter]]): +def test_command(request_id: str, params: Optional[List[CommandParameter]]) -> None: request_params: Dict[str, Any] = ( {} if params is None else {x.name: None for x in params} ) @@ -62,7 +62,7 @@ def test_command(request_id: str, params: Optional[List[CommandParameter]]): @pytest.mark.parametrize("params", PARAMS) def test_msgpack_serialization_deserialization( request_id: str, params: Optional[List[CommandParameter]] -): +) -> None: request_params: Dict[str, Any] = ( {} if params is None else {x.name: None for x in params} @@ -91,20 +91,20 @@ def test_msgpack_serialization_deserialization( @pytest.mark.parametrize("payload", [None, "a_string", 1, 1.0, b"some_bytes"]) def test_command_reply_payloads_serialization_deserialization( request_id: str, payload: Any -): +) -> None: command_reply = CommandReply(reply_id=request_id, payload=payload) assert command_reply assert command_reply == CommandReply.from_bytes(command_reply.to_bytes()) -def test_command_accepted_ok(request_id: str): +def test_command_accepted_ok(request_id: str) -> None: assert CommandReceived(request_id=request_id, accepted=True, error_message=None) assert CommandReceived( request_id=request_id, accepted=False, error_message="some error" ) -def test_command_accepted_fails(request_id: str): +def test_command_accepted_fails(request_id: str) -> None: with pytest.raises(ValidationError): assert CommandReceived( request_id=request_id, accepted=False, error_message=None @@ -115,7 +115,7 @@ def test_command_accepted_fails(request_id: str): ) -def test_duplicate_command_parameter_name_in_manifest(): +def test_duplicate_command_parameter_name_in_manifest() -> None: with pytest.raises(ValidationError): CommandManifest( action="test", diff --git a/tests/test_transport.py b/tests/test_transport.py index 459b2c6..623ad1d 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,7 +1,9 @@ from typing import Iterable +from typing import Optional from typing import Type import pytest +from _pytest.fixtures import SubRequest from osparc_control.transport.base_transport import BaseTransport from osparc_control.transport.base_transport import SenderReceiverPair @@ -21,8 +23,8 @@ def _payload_generator(start: int, stop: int) -> Iterable[bytes]: @pytest.fixture(params=[InMemoryTransport, ZeroMQTransport]) -def transport_class(request) -> Type[BaseTransport]: - return request.param +def transport_class(request: SubRequest) -> Type[BaseTransport]: + return request.param # type: ignore @pytest.fixture @@ -30,14 +32,14 @@ def sender_receiver_pair( transport_class: Type[BaseTransport], ) -> Iterable[SenderReceiverPair]: if transport_class == InMemoryTransport: - sender = transport_class("A", "B") - receiver = transport_class("B", "A") + sender = transport_class("A", "B") # type: ignore + receiver = transport_class("B", "A") # type: ignore elif transport_class == ZeroMQTransport: port = 1111 - sender = transport_class( + sender = transport_class( # type: ignore listen_port=port, remote_host="localhost", remote_port=port ) - receiver = transport_class( + receiver = transport_class( # type: ignore listen_port=port, remote_host="localhost", remote_port=port ) @@ -52,21 +54,21 @@ def sender_receiver_pair( sender_receiver_pair.receiver_cleanup() -def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair): +def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair) -> None: for message in _payload_generator(1, 10): print("sending", message) sender_receiver_pair.send_bytes(message) for expected_message in _payload_generator(1, 10): - message = sender_receiver_pair.receive_bytes() - print("received", message) - assert message == expected_message + received_message: Optional[bytes] = sender_receiver_pair.receive_bytes() + print("received", received_message) + assert received_message == expected_message -def test_receive_nothing(sender_receiver_pair: SenderReceiverPair): +def test_receive_nothing(sender_receiver_pair: SenderReceiverPair) -> None: assert sender_receiver_pair.receive_bytes() is None -def test_receive_returns_none_if_no_message_available(): +def test_receive_returns_none_if_no_message_available() -> None: receiver = InMemoryTransport("B", "A") assert receiver.receive_bytes() is None From 9d89009fd38e07bb3ffa8d5e76789af57060a63f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:41:46 +0100 Subject: [PATCH 36/86] trying to fix mypy errors --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index d1c1eb0..6f7a318 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,7 @@ from pathlib import Path from textwrap import dedent -import nox +import nox # type: ignore try: from nox_poetry import Session, session From 4afb8cd446c4f12a80f76a9e491d73165a044a1b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:51:28 +0100 Subject: [PATCH 37/86] fix mypy locally --- Makefile | 2 +- noxfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 20c5581..f71eb52 100644 --- a/Makefile +++ b/Makefile @@ -39,4 +39,4 @@ codestyle: # runs codestyle enforcement .PHONY: mypy mypy: # runs mypy - mypy src tests docs/conf.py + nox -p 3.6 -r -s mypy diff --git a/noxfile.py b/noxfile.py index 6f7a318..d1c1eb0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,7 @@ from pathlib import Path from textwrap import dedent -import nox # type: ignore +import nox try: from nox_poetry import Session, session From e49dda603f3691abb1f1c01d5cd80f5d7c553e17 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:54:27 +0100 Subject: [PATCH 38/86] making docs build on py.36 in CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0fc35b..0da19ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: - { python: "3.10", os: "macos-latest", session: "tests" } - { python: "3.10", os: "ubuntu-latest", session: "typeguard" } - { python: "3.10", os: "ubuntu-latest", session: "xdoctest" } - - { python: "3.10", os: "ubuntu-latest", session: "docs-build" } + - { python: "3.6", os: "ubuntu-latest", session: "docs-build" } env: NOXSESSION: ${{ matrix.session }} From db24b5df3ff2fbe830e01ead534368d927584cdb Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:55:52 +0100 Subject: [PATCH 39/86] fix CI for noxfile --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index d1c1eb0..4257e71 100644 --- a/noxfile.py +++ b/noxfile.py @@ -122,7 +122,7 @@ def mypy(session: Session) -> None: session.install("mypy", "pytest") session.run("mypy", *args) if not session.posargs: - session.run("mypy", f"--python-executable={sys.executable}", "noxfile.py") + session.run("mypy", "noxfile.py") @session(python=python_versions) From b7752b893afa76fb0adcc4818ac7451e53ffe4e9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 13:59:16 +0100 Subject: [PATCH 40/86] this should fix typing issues with noxfile --- noxfile.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/noxfile.py b/noxfile.py index 4257e71..cc91fe2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,10 +5,10 @@ from pathlib import Path from textwrap import dedent -import nox +import nox # type: ignore try: - from nox_poetry import Session, session + from nox_poetry import Session, session # type: ignore except ImportError: message = f"""\ Nox failed to import the 'nox-poetry' package. @@ -83,7 +83,7 @@ def activate_virtualenv_in_precommit_hooks(session: Session) -> None: hook.write_text("\n".join(lines)) -@session(name="pre-commit", python="3.10") +@session(name="pre-commit", python="3.10") # type: ignore def precommit(session: Session) -> None: """Lint using pre-commit.""" args = session.posargs or ["run", "--all-files", "--show-diff-on-failure"] @@ -106,7 +106,7 @@ def precommit(session: Session) -> None: activate_virtualenv_in_precommit_hooks(session) -@session(python="3.10") +@session(python="3.10") # type: ignore def safety(session: Session) -> None: """Scan dependencies for insecure packages.""" requirements = session.poetry.export_requirements() @@ -114,7 +114,7 @@ def safety(session: Session) -> None: session.run("safety", "check", "--full-report", f"--file={requirements}") -@session(python=python_versions) +@session(python=python_versions) # type: ignore def mypy(session: Session) -> None: """Type-check using mypy.""" args = session.posargs or ["src", "tests", "docs/conf.py"] @@ -125,7 +125,7 @@ def mypy(session: Session) -> None: session.run("mypy", "noxfile.py") -@session(python=python_versions) +@session(python=python_versions) # type: ignore def tests(session: Session) -> None: """Run the test suite.""" session.install(".") @@ -137,7 +137,7 @@ def tests(session: Session) -> None: session.notify("coverage", posargs=[]) -@session +@session # type: ignore def coverage(session: Session) -> None: """Produce the coverage report.""" args = session.posargs or ["report"] @@ -150,7 +150,7 @@ def coverage(session: Session) -> None: session.run("coverage", *args) -@session(python=python_versions) +@session(python=python_versions) # type: ignore def typeguard(session: Session) -> None: """Runtime type checking using Typeguard.""" session.install(".") @@ -158,7 +158,7 @@ def typeguard(session: Session) -> None: session.run("pytest", f"--typeguard-packages={package}", *session.posargs) -@session(python=python_versions) +@session(python=python_versions) # type: ignore def xdoctest(session: Session) -> None: """Run examples with xdoctest.""" if session.posargs: @@ -173,7 +173,7 @@ def xdoctest(session: Session) -> None: session.run("python", "-m", "xdoctest", *args) -@session(name="docs-build", python="3.6") +@session(name="docs-build", python="3.6") # type: ignore def docs_build(session: Session) -> None: """Build the documentation.""" args = session.posargs or ["docs", "docs/_build"] @@ -190,7 +190,7 @@ def docs_build(session: Session) -> None: session.run("sphinx-build", *args) -@session(python="3.6") +@session(python="3.6") # type: ignore def docs(session: Session) -> None: """Build and serve the documentation with live reloading on file changes.""" args = session.posargs or [ From 7b24392f2339dccfc655a6c5c743999789221a2b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 14:07:01 +0100 Subject: [PATCH 41/86] fixed test_transport --- tests/test_transport.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_transport.py b/tests/test_transport.py index 623ad1d..b5b8956 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -31,10 +31,14 @@ def transport_class(request: SubRequest) -> Type[BaseTransport]: def sender_receiver_pair( transport_class: Type[BaseTransport], ) -> Iterable[SenderReceiverPair]: + sender: Optional[BaseTransport] = None + receiver: Optional[BaseTransport] = None + if transport_class == InMemoryTransport: sender = transport_class("A", "B") # type: ignore receiver = transport_class("B", "A") # type: ignore - elif transport_class == ZeroMQTransport: + + if transport_class == ZeroMQTransport: port = 1111 sender = transport_class( # type: ignore listen_port=port, remote_host="localhost", remote_port=port @@ -43,6 +47,8 @@ def sender_receiver_pair( listen_port=port, remote_host="localhost", remote_port=port ) + assert sender + assert receiver sender_receiver_pair = SenderReceiverPair(sender=sender, receiver=receiver) sender_receiver_pair.sender_init() From 2081bfef179f22d3bd988d7ad85cbfd3c833aec3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 14:37:20 +0100 Subject: [PATCH 42/86] adding base test_core --- tests/test_core.py | 144 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/test_core.py diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..e19f1c5 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,144 @@ +import random +import time +from typing import Iterable +from typing import List + +import pytest + +from osparc_control.core import ControlInterface +from osparc_control.models import CommandManifest +from osparc_control.models import CommandParameter +from osparc_control.models import CommnadType + +WAIT_FOR_DELIVERY = 0.01 + + +def _get_control_interface( + local_port: int, remote_port: int, exposed_interface: List[CommandManifest] +) -> ControlInterface: + return ControlInterface( + remote_host="localhost", + exposed_interface=exposed_interface, + remote_port=remote_port, + listen_port=local_port, + ) + + +@pytest.fixture +def control_interface_a() -> Iterable[ControlInterface]: + control_interface = _get_control_interface(1234, 1235, []) + control_interface.start_background_sync() + yield control_interface + control_interface.stop_background_sync() + + +@pytest.fixture +def mainfest_b() -> List[CommandManifest]: + add_numbers = CommandManifest( + action="add_numbers", + description="adds two numbers", + params=[ + CommandParameter(name="a", description="param to add"), + CommandParameter(name="b", description="param to add"), + ], + command_type=CommnadType.WITH_DELAYED_REPLY, + ) + + get_random = CommandManifest( + action="get_random", + description="returns a random number", + params=[], + command_type=CommnadType.WITH_IMMEDIATE_REPLY, + ) + + greet_user = CommandManifest( + action="greet_user", + description="prints the status of the solver", + params=[CommandParameter(name="name", description="name to greet")], + command_type=CommnadType.WITHOUT_REPLY, + ) + + return [add_numbers, get_random, greet_user] + + +@pytest.fixture +def control_interface_b( + mainfest_b: List[CommandManifest], +) -> Iterable[ControlInterface]: + control_interface = _get_control_interface(1235, 1234, mainfest_b) + control_interface.start_background_sync() + yield control_interface + control_interface.stop_background_sync() + + +def test_request_with_delayed_reply( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + # SIDE A + request_id = control_interface_a.request_with_delayed_reply( + "add_numbers", params={"a": 10, "b": 13.3} + ) + + # SIDE B + wait_for_requests = True + while wait_for_requests: + for command in control_interface_b.get_incoming_requests(): + assert command.action == "add_numbers" + control_interface_b.reply_to_command( + request_id=command.request_id, payload=sum(command.params.values()) + ) + wait_for_requests = False + + # SIDE A + time.sleep(WAIT_FOR_DELIVERY) + + has_result, result = control_interface_a.check_for_reply(request_id=request_id) + assert has_result is True + assert result is not None + + +def test_request_with_immediate_reply( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + def _worker_b() -> None: + wait_for_requests = True + while wait_for_requests: + for command in control_interface_b.get_incoming_requests(): + assert command.action == "get_random" + control_interface_b.reply_to_command( + request_id=command.request_id, + payload=random.randint(1, 1000), # noqa: S311 + ) + wait_for_requests = False + + from threading import Thread + + thread = Thread(target=_worker_b, daemon=True) + thread.start() + + random_integer = control_interface_a.request_with_immediate_reply( + "get_random", timeout=1.0 + ) + assert type(random_integer) == int + assert random_integer + assert 1 <= random_integer <= 1000 + + thread.join() + + +def test_request_without_reply( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + # SIDE A + + control_interface_a.request_without_reply("greet_user", params={"name": "tester"}) + expected_message = "hello tester" + + # SIDE B + wait_for_requests = True + while wait_for_requests: + for command in control_interface_b.get_incoming_requests(): + assert command.action == "greet_user" + message = f"hello {command.params['name']}" + assert message == expected_message + wait_for_requests = False From 97338b9030e25f5e5c9c015ac8826a9fc8c96298 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 15:36:24 +0100 Subject: [PATCH 43/86] fixed coverage --- poetry.lock | 8 ++-- pyproject.toml | 5 ++- src/osparc_control/core.py | 11 +++--- tests/test_core.py | 75 +++++++++++++++++++++++++++++++++++++- 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index 030e951..d03b114 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1082,14 +1082,14 @@ python-versions = "*" [[package]] name = "urllib3" -version = "1.26.8" +version = "1.26.9" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] @@ -1930,8 +1930,8 @@ u-msgpack-python = [ {file = "u_msgpack_python-2.7.1-py2.py3-none-any.whl", hash = "sha256:0eb339ae27ec3085945244d17b74fd1ed875e866974d63caaa85d90fca9060a7"}, ] urllib3 = [ - {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, - {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] virtualenv = [ {file = "virtualenv-20.13.3-py2.py3-none-any.whl", hash = "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021"}, diff --git a/pyproject.toml b/pyproject.toml index 29d36a3..7a74317 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,11 @@ branch = true source = ["osparc_control", "tests"] [tool.coverage.report] +exclude_lines = [ + "pragma: no cover" +] show_missing = true -fail_under = 100 +fail_under = 99 [tool.mypy] strict = true diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index e7e7bab..d9ddf36 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -14,7 +14,6 @@ from uuid import uuid4 from pydantic import ValidationError -from tenacity import RetryError from tenacity import Retrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed @@ -123,7 +122,7 @@ def _sender_worker(self) -> None: def _handle_command_request(self, response: bytes) -> None: command_request: Optional[CommandRequest] = CommandRequest.from_bytes(response) if command_request is None: - return + return # pragma: no cover def _refuse_and_return(error_message: str) -> None: self._out_queue.put( @@ -255,7 +254,7 @@ def _enqueue_call( ): with attempt: command_received = self._incoming_command_queue.get(block=False) - except RetryError: + except Empty: raise NoCommandReceivedArrivedError() from None assert command_received # noqa: S101 @@ -310,13 +309,13 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: request_id, None ) if tracked_request is None: - return False, None + return False, None # pragma: no cover # check for the correct type of request if tracked_request.request.command_type not in { CommnadType.WITH_IMMEDIATE_REPLY, CommnadType.WITH_DELAYED_REPLY, }: - raise RuntimeError( + raise RuntimeError( # pragma: no cover f"Request {tracked_request.request} not expect a " f"reply, found reply {tracked_request.reply}" ) @@ -352,7 +351,7 @@ def request_with_immediate_reply( reply_received, result = self.check_for_reply(request.request_id) if not reply_received: raise NoReplyError() - except RetryError: + except NoReplyError: pass return result diff --git a/tests/test_core.py b/tests/test_core.py index e19f1c5..17b948b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5,13 +5,24 @@ import pytest +import osparc_control from osparc_control.core import ControlInterface +from osparc_control.errors import CommnadNotAcceptedError +from osparc_control.errors import NoCommandReceivedArrivedError from osparc_control.models import CommandManifest from osparc_control.models import CommandParameter from osparc_control.models import CommnadType WAIT_FOR_DELIVERY = 0.01 +ALL_COMMAND_TYPES: List[CommnadType] = [ + CommnadType.WITH_DELAYED_REPLY, + CommnadType.WITH_DELAYED_REPLY, + CommnadType.WITHOUT_REPLY, +] + +# UTILS + def _get_control_interface( local_port: int, remote_port: int, exposed_interface: List[CommandManifest] @@ -24,6 +35,9 @@ def _get_control_interface( ) +# FIXTURES + + @pytest.fixture def control_interface_a() -> Iterable[ControlInterface]: control_interface = _get_control_interface(1234, 1235, []) @@ -71,6 +85,17 @@ def control_interface_b( control_interface.stop_background_sync() +@pytest.fixture +def mock_wait_for_received() -> Iterable[None]: + previous = osparc_control.core.WAIT_FOR_RECEIVED + osparc_control.core.WAIT_FOR_RECEIVED = 0.01 + yield + osparc_control.core.WAIT_FOR_RECEIVED = previous + + +# TESTS + + def test_request_with_delayed_reply( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: @@ -101,6 +126,7 @@ def test_request_with_immediate_reply( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: def _worker_b() -> None: + count = 1 wait_for_requests = True while wait_for_requests: for command in control_interface_b.get_incoming_requests(): @@ -109,7 +135,9 @@ def _worker_b() -> None: request_id=command.request_id, payload=random.randint(1, 1000), # noqa: S311 ) - wait_for_requests = False + count += 1 + + wait_for_requests = count > 2 from threading import Thread @@ -123,6 +151,11 @@ def _worker_b() -> None: assert random_integer assert 1 <= random_integer <= 1000 + no_reply_in_time = control_interface_a.request_with_immediate_reply( + "get_random", timeout=0.001 + ) + assert no_reply_in_time is None + thread.join() @@ -142,3 +175,43 @@ def test_request_without_reply( message = f"hello {command.params['name']}" assert message == expected_message wait_for_requests = False + + +@pytest.mark.parametrize("command_type", ALL_COMMAND_TYPES) +def test_no_same_action_command_in_exposed_interface(command_type: CommnadType) -> None: + test_command_manifest = CommandManifest( + action="test", description="test", params=[], command_type=command_type + ) + + with pytest.raises(ValueError): + _get_control_interface(100, 100, [test_command_manifest, test_command_manifest]) + + +def test_no_registered_command( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + with pytest.raises(CommnadNotAcceptedError): + control_interface_a.request_without_reply("command_not_defined") + + +def test_wrong_command_type( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + with pytest.raises(CommnadNotAcceptedError): + control_interface_a.request_without_reply("add_numbers") + + +def test_command_params_mismatch( + control_interface_a: ControlInterface, control_interface_b: ControlInterface +) -> None: + with pytest.raises(CommnadNotAcceptedError): + control_interface_a.request_without_reply("add_numbers", {"nope": 123}) + + +def test_side_b_does_not_reply_in_time(mock_wait_for_received: None): + control_interface = _get_control_interface(8263, 8263, []) + control_interface.start_background_sync() + with pytest.raises(NoCommandReceivedArrivedError): + control_interface.request_without_reply( + "no_remote_side_for_command", {"nope": 123} + ) From fd168269a84b174469945cd020c2b3bb35f4a28c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 15:38:27 +0100 Subject: [PATCH 44/86] fixed mypy --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 17b948b..382ce56 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -208,7 +208,7 @@ def test_command_params_mismatch( control_interface_a.request_without_reply("add_numbers", {"nope": 123}) -def test_side_b_does_not_reply_in_time(mock_wait_for_received: None): +def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: control_interface = _get_control_interface(8263, 8263, []) control_interface.start_background_sync() with pytest.raises(NoCommandReceivedArrivedError): From faac9f5463bdd8b75635c19b93a1b9f4fedbe6d6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 15:43:47 +0100 Subject: [PATCH 45/86] raised timeout for macos in CI --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 382ce56..89adeb7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -13,7 +13,7 @@ from osparc_control.models import CommandParameter from osparc_control.models import CommnadType -WAIT_FOR_DELIVERY = 0.01 +WAIT_FOR_DELIVERY = 0.1 ALL_COMMAND_TYPES: List[CommnadType] = [ CommnadType.WITH_DELAYED_REPLY, From 3d70110162fbed1b4443c9996771631cac031265 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 15:51:50 +0100 Subject: [PATCH 46/86] updated target code coverage --- codecov.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 9ac2650..248ee86 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,7 +3,7 @@ coverage: status: project: default: - target: "100" + target: "99" patch: default: - target: "100" + target: "99" From abc05776fe6301b80cb4df887b5448bc2ce7ee1a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 16:35:54 +0100 Subject: [PATCH 47/86] added some examples --- README.rst | 8 +------- docs/reference.rst | 15 ++++++++++++++- docs/usage.rst | 20 +++++++++++++++++--- examples/base_time_add/README.md | 4 ---- examples/base_time_add/sidecar_controller.py | 3 +++ src/osparc_control/core.py | 5 ++++- 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 19c3196..0f74e28 100644 --- a/README.rst +++ b/README.rst @@ -42,12 +42,6 @@ Features * TODO -Requirements ------------- - -* TODO - - Installation ------------ @@ -61,7 +55,7 @@ You can install *Osparc Control* via pip_ from PyPI_: Usage ----- -Please see the `Command-line Reference `_ for details. +Please have a look at the examples folder. Contributing diff --git a/docs/reference.rst b/docs/reference.rst index 0266d12..b6ba8ff 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -5,5 +5,18 @@ Reference osparc_control -------------- -.. automodule:: osparc_control +.. autoclass:: osparc_control.ControlInterface :members: + :undoc-members: + +.. autoclass:: osparc_control.CommandManifest + :members: + :undoc-members: + +.. autoclass:: osparc_control.CommandParameter + :members: + :undoc-members: + +.. autoclass:: osparc_control.CommnadType + :members: + :undoc-members: diff --git a/docs/usage.rst b/docs/usage.rst index 6b19660..b088d80 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,6 +1,20 @@ Usage ===== -.. click:: osparc_control.__main__:main - :prog: osparc-control - :nested: full +Please have a look at the examples folder. + + +Minimual examples +================= + +In one terminal run ``python examples/basic/requester.py``: + +.. literalinclude:: ../examples/basic/requester.py + :language: python + + + +In a second terminal run ``python examples/basic/replyer.py``: + +.. literalinclude:: ../examples/basic/replyer.py + :language: python diff --git a/examples/base_time_add/README.md b/examples/base_time_add/README.md index 3aabad0..98ff72a 100644 --- a/examples/base_time_add/README.md +++ b/examples/base_time_add/README.md @@ -6,7 +6,3 @@ This example consists of a `time_solver`. Which can add, the current time by a p In one terminal run `sidecar_controller.py`. In a second terminal run `time_solver.py`. It will load data from the `sidecar_solver.py` to use when communicating with the `sidecar_controller.py` - -# In this example - -Only the `solver` exposes an interface that can be queried. The `controller` does not have an exposed interface. diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py index bddfd94..270c525 100644 --- a/examples/base_time_add/sidecar_controller.py +++ b/examples/base_time_add/sidecar_controller.py @@ -25,6 +25,9 @@ print("getting solver time") solver_time = control_interface.request_with_immediate_reply("get_time", timeout=1.0) +random_int = control_interface.request_with_immediate_reply( + "random_in_range", timeout=1.0, params={"a": 1, "b": 3} +) print("solver time", solver_time) print("sending command to print internal status") diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index d9ddf36..1905b3e 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -357,7 +357,10 @@ def request_with_immediate_reply( return result def get_incoming_requests(self) -> List[CommandRequest]: - """retruns all accumulated CommandRequests""" + """ + Non blocking, retruns all accumulated CommandRequests. + It is meant to be used in an existing cycle + """ results: Deque[CommandRequest] = deque() # fetch all elements empty From 411b0e37c007cd6c0ef587a440bb79ec6f5dbfec Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 16 Mar 2022 16:38:28 +0100 Subject: [PATCH 48/86] added missing example files --- examples/basic/replyer.py | 42 +++++++++++++++++++++++++++++++++++++ examples/basic/requester.py | 15 +++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 examples/basic/replyer.py create mode 100644 examples/basic/requester.py diff --git a/examples/basic/replyer.py b/examples/basic/replyer.py new file mode 100644 index 0000000..02146d7 --- /dev/null +++ b/examples/basic/replyer.py @@ -0,0 +1,42 @@ +import random +import time + +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommnadType +from osparc_control import ControlInterface + +random_in_range_manifest = CommandManifest( + action="random_in_range", + description="gets the time", + params=[ + CommandParameter(name="a", description="lower bound for random numbers"), + CommandParameter(name="b", description="upper bound for random numbers"), + ], + command_type=CommnadType.WITH_IMMEDIATE_REPLY, +) + +control_interface = ControlInterface( + remote_host="localhost", + exposed_interface=[random_in_range_manifest], + remote_port=2346, + listen_port=2345, +) +control_interface.start_background_sync() + +wait_for_requests = True +while wait_for_requests: + for command in control_interface.get_incoming_requests(): + if command.action == random_in_range_manifest.action: + random_int = random.randint( # noqa: S311 + command.params["a"], command.params["b"] + ) + control_interface.reply_to_command( + request_id=command.request_id, payload=random_int + ) + wait_for_requests = False + +# allow for message to be delivered +time.sleep(0.01) + +control_interface.stop_background_sync() diff --git a/examples/basic/requester.py b/examples/basic/requester.py new file mode 100644 index 0000000..fa34c79 --- /dev/null +++ b/examples/basic/requester.py @@ -0,0 +1,15 @@ +from osparc_control import ControlInterface + +control_interface = ControlInterface( + remote_host="localhost", exposed_interface=[], remote_port=2345, listen_port=2346 +) + +control_interface.start_background_sync() + +random_int = control_interface.request_with_immediate_reply( + "random_in_range", timeout=10.0, params={"a": 1, "b": 10} +) +print(random_int) +assert 1 <= random_int <= 10 # noqa: S101 + +control_interface.stop_background_sync() From 206a67344b0373db3efcd26794385a8d0ab06b0b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 17 Mar 2022 08:09:51 +0100 Subject: [PATCH 49/86] bumped version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7a74317..62bccbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "osparc-control" -version = "0.0.0" +version = "0.0.1" description = "Osparc Control" authors = ["Andrei Neagu "] license = "MIT" From 29ac0f8b3803372dc9fb745d587bfced40b0c851 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 17 Mar 2022 08:10:09 +0100 Subject: [PATCH 50/86] refactor docs --- src/osparc_control/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index a376450..5ec0f42 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -41,12 +41,13 @@ class CommnadType(str, Enum): WITHOUT_REPLY = "WITHOUT_REPLY" # the command will provide a reply - # the suer is require to check for the results + # the user is require to check for the results # of this reply WITH_DELAYED_REPLY = "WITH_DELAYED_REPLY" # the command will return the result immediately # and user code will be blocked until reply arrives + # used for very fast replies (provide data which already exists) WITH_IMMEDIATE_REPLY = "WITH_IMMEDIATE_REPLY" From e1ace8139f1d588874b347a41b4336ae5025b2c3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 17 Mar 2022 10:45:55 +0100 Subject: [PATCH 51/86] fix typo --- docs/reference.rst | 2 +- examples/base_time_add/sidecar_solver.py | 8 ++++---- examples/basic/replyer.py | 4 ++-- src/osparc_control/__init__.py | 4 ++-- src/osparc_control/core.py | 14 +++++++------- src/osparc_control/models.py | 6 +++--- tests/test_core.py | 18 +++++++++--------- tests/test_models.py | 10 +++++----- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index b6ba8ff..6694704 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -17,6 +17,6 @@ osparc_control :members: :undoc-members: -.. autoclass:: osparc_control.CommnadType +.. autoclass:: osparc_control.CommandType :members: :undoc-members: diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py index af0676f..7afe005 100644 --- a/examples/base_time_add/sidecar_solver.py +++ b/examples/base_time_add/sidecar_solver.py @@ -1,6 +1,6 @@ from osparc_control import CommandManifest from osparc_control import CommandParameter -from osparc_control import CommnadType +from osparc_control import CommandType from osparc_control import ControlInterface @@ -10,21 +10,21 @@ params=[ CommandParameter(name="a", description="param to add to internal time"), ], - command_type=CommnadType.WITH_DELAYED_REPLY, + command_type=CommandType.WITH_DELAYED_REPLY, ) command_get_time = CommandManifest( action="get_time", description="gets the time", params=[], - command_type=CommnadType.WITH_IMMEDIATE_REPLY, + command_type=CommandType.WITH_IMMEDIATE_REPLY, ) command_print_solver_status = CommandManifest( action="print_status", description="prints the status of the solver", params=[], - command_type=CommnadType.WITHOUT_REPLY, + command_type=CommandType.WITHOUT_REPLY, ) diff --git a/examples/basic/replyer.py b/examples/basic/replyer.py index 02146d7..5226bd4 100644 --- a/examples/basic/replyer.py +++ b/examples/basic/replyer.py @@ -3,7 +3,7 @@ from osparc_control import CommandManifest from osparc_control import CommandParameter -from osparc_control import CommnadType +from osparc_control import CommandType from osparc_control import ControlInterface random_in_range_manifest = CommandManifest( @@ -13,7 +13,7 @@ CommandParameter(name="a", description="lower bound for random numbers"), CommandParameter(name="b", description="upper bound for random numbers"), ], - command_type=CommnadType.WITH_IMMEDIATE_REPLY, + command_type=CommandType.WITH_IMMEDIATE_REPLY, ) control_interface = ControlInterface( diff --git a/src/osparc_control/__init__.py b/src/osparc_control/__init__.py index b80c2ed..a810ad0 100644 --- a/src/osparc_control/__init__.py +++ b/src/osparc_control/__init__.py @@ -2,6 +2,6 @@ from .core import ControlInterface from .models import CommandManifest from .models import CommandParameter -from .models import CommnadType +from .models import CommandType -__all__ = ["ControlInterface", "CommandManifest", "CommandParameter", "CommnadType"] +__all__ = ["ControlInterface", "CommandManifest", "CommandParameter", "CommandType"] diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 1905b3e..a1873e6 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -25,7 +25,7 @@ from .models import CommandReceived from .models import CommandReply from .models import CommandRequest -from .models import CommnadType +from .models import CommandType from .models import RequestsTracker from .models import TrackedRequest from .transport.base_transport import SenderReceiverPair @@ -228,7 +228,7 @@ def _enqueue_call( self, action: str, params: Optional[Dict[str, Any]], - expected_command_type: CommnadType, + expected_command_type: CommandType, ) -> CommandRequest: """validates and enqueues the call for delivery to remote""" request = CommandRequest( @@ -284,7 +284,7 @@ def request_without_reply( self, action: str, params: Optional[Dict[str, Any]] = None ) -> None: """No reply will be provided by remote side for this command""" - self._enqueue_call(action, params, CommnadType.WITHOUT_REPLY) + self._enqueue_call(action, params, CommandType.WITHOUT_REPLY) def request_with_delayed_reply( self, action: str, params: Optional[Dict[str, Any]] = None @@ -293,7 +293,7 @@ def request_with_delayed_reply( returns a `request_id` to be used with `check_for_reply` to monitor if a reply to the request was returned. """ - request = self._enqueue_call(action, params, CommnadType.WITH_DELAYED_REPLY) + request = self._enqueue_call(action, params, CommandType.WITH_DELAYED_REPLY) return request.request_id def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: @@ -312,8 +312,8 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: return False, None # pragma: no cover # check for the correct type of request if tracked_request.request.command_type not in { - CommnadType.WITH_IMMEDIATE_REPLY, - CommnadType.WITH_DELAYED_REPLY, + CommandType.WITH_IMMEDIATE_REPLY, + CommandType.WITH_DELAYED_REPLY, }: raise RuntimeError( # pragma: no cover f"Request {tracked_request.request} not expect a " @@ -337,7 +337,7 @@ def request_with_immediate_reply( A timeout for this function is required. If the timeout is reached `None` will be returned. """ - request = self._enqueue_call(action, params, CommnadType.WITH_IMMEDIATE_REPLY) + request = self._enqueue_call(action, params, CommandType.WITH_IMMEDIATE_REPLY) result: Optional[Any] = None diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 5ec0f42..788ee43 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -36,7 +36,7 @@ class CommandParameter(BaseModel): ) -class CommnadType(str, Enum): +class CommandType(str, Enum): # the command expects no reply WITHOUT_REPLY = "WITHOUT_REPLY" @@ -60,7 +60,7 @@ class CommandManifest(BaseModel): params: List[CommandParameter] = Field( None, description="requested parameters by the user" ) - command_type: CommnadType = Field( + command_type: CommandType = Field( ..., description="describes the command type, behaviour and usage" ) @@ -78,7 +78,7 @@ class CommandRequest(CommandBase): request_id: str = Field(..., description="unique identifier") action: str = Field(..., description="name of the action to be triggered on remote") params: Dict[str, Any] = Field({}, description="requested parameters by the user") - command_type: CommnadType = Field( + command_type: CommandType = Field( ..., description="describes the command type, behaviour and usage" ) diff --git a/tests/test_core.py b/tests/test_core.py index 89adeb7..e9d64ba 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -11,14 +11,14 @@ from osparc_control.errors import NoCommandReceivedArrivedError from osparc_control.models import CommandManifest from osparc_control.models import CommandParameter -from osparc_control.models import CommnadType +from osparc_control.models import CommandType WAIT_FOR_DELIVERY = 0.1 -ALL_COMMAND_TYPES: List[CommnadType] = [ - CommnadType.WITH_DELAYED_REPLY, - CommnadType.WITH_DELAYED_REPLY, - CommnadType.WITHOUT_REPLY, +ALL_COMMAND_TYPES: List[CommandType] = [ + CommandType.WITH_DELAYED_REPLY, + CommandType.WITH_DELAYED_REPLY, + CommandType.WITHOUT_REPLY, ] # UTILS @@ -55,21 +55,21 @@ def mainfest_b() -> List[CommandManifest]: CommandParameter(name="a", description="param to add"), CommandParameter(name="b", description="param to add"), ], - command_type=CommnadType.WITH_DELAYED_REPLY, + command_type=CommandType.WITH_DELAYED_REPLY, ) get_random = CommandManifest( action="get_random", description="returns a random number", params=[], - command_type=CommnadType.WITH_IMMEDIATE_REPLY, + command_type=CommandType.WITH_IMMEDIATE_REPLY, ) greet_user = CommandManifest( action="greet_user", description="prints the status of the solver", params=[CommandParameter(name="name", description="name to greet")], - command_type=CommnadType.WITHOUT_REPLY, + command_type=CommandType.WITHOUT_REPLY, ) return [add_numbers, get_random, greet_user] @@ -178,7 +178,7 @@ def test_request_without_reply( @pytest.mark.parametrize("command_type", ALL_COMMAND_TYPES) -def test_no_same_action_command_in_exposed_interface(command_type: CommnadType) -> None: +def test_no_same_action_command_in_exposed_interface(command_type: CommandType) -> None: test_command_manifest = CommandManifest( action="test", description="test", params=[], command_type=command_type ) diff --git a/tests/test_models.py b/tests/test_models.py index e2cd372..49ea1ff 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -13,7 +13,7 @@ from osparc_control.models import CommandReceived from osparc_control.models import CommandReply from osparc_control.models import CommandRequest -from osparc_control.models import CommnadType +from osparc_control.models import CommandType @pytest.fixture @@ -30,7 +30,7 @@ def request_id() -> str: @pytest.mark.parametrize("params", PARAMS) def test_command_manifest(params: Optional[List[CommandParameter]]) -> None: - for command_type in CommnadType: + for command_type in CommandType: assert CommandManifest( action="test", description="some test action", @@ -47,7 +47,7 @@ def test_command(request_id: str, params: Optional[List[CommandParameter]]) -> N manifest = CommandManifest( action="test", description="some test action", - command_type=CommnadType.WITHOUT_REPLY, + command_type=CommandType.WITHOUT_REPLY, params=[] if params is None else params, ) @@ -70,7 +70,7 @@ def test_msgpack_serialization_deserialization( manifest = CommandManifest( action="test", description="some test action", - command_type=CommnadType.WITH_IMMEDIATE_REPLY, + command_type=CommandType.WITH_IMMEDIATE_REPLY, params=[] if params is None else params, ) @@ -124,5 +124,5 @@ def test_duplicate_command_parameter_name_in_manifest() -> None: CommandParameter(name="a", description="ok"), CommandParameter(name="a", description="not allowed same name"), ], - command_type=CommnadType.WITH_DELAYED_REPLY, + command_type=CommandType.WITH_DELAYED_REPLY, ) From e5e2825cb29034e4210d1ad47c9364ec1b6c09e5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:03:35 +0200 Subject: [PATCH 52/86] fix readme --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f71eb52..8f50338 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Heps setup basic env development +# Helps setup basic env development # poetry is required on your system # suggested installation method @@ -8,7 +8,7 @@ install-poetry: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - -# install deve dependecis as sugested by coockicutter +# install development dependencies as suggested by cookiecutter # https://cookiecutter-hypermodern-python.readthedocs.io/en/2021.11.26/quickstart.html .PHONY: install-dev install-dev: From d21022ce5c8f9158bb0827e70692f34ff90fd6ef Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:13:16 +0200 Subject: [PATCH 53/86] added context manager support --- src/osparc_control/core.py | 7 +++++++ tests/test_core.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index a1873e6..17bd199 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -103,6 +103,13 @@ def _process_manifest(manifest: CommandManifest) -> CommandManifest: ) self._continue: bool = True + def __enter__(self) -> "ControlInterface": + self.start_background_sync() + return self + + def __exit__(self, *args: Any) -> None: + self.stop_background_sync() + def _sender_worker(self) -> None: self._sender_receiver_pair.sender_init() diff --git a/tests/test_core.py b/tests/test_core.py index e9d64ba..390f7b4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -96,6 +96,11 @@ def mock_wait_for_received() -> Iterable[None]: # TESTS +def test_context_manager(mainfest_b: List[CommandManifest]) -> None: + with _get_control_interface(1235, 1234, mainfest_b): + pass + + def test_request_with_delayed_reply( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: From 7d0d7aaa851e89e9f4adffe58f9b177c6f365236 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:22:24 +0200 Subject: [PATCH 54/86] refacto rename --- src/osparc_control/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 17bd199..7779e40 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -73,13 +73,13 @@ def __init__( remote_host=remote_host, remote_port=remote_port, listen_port=listen_port ) - def _process_manifest(manifest: CommandManifest) -> CommandManifest: + def _update_remapped_params(manifest: CommandManifest) -> CommandManifest: manifest._remapped_params = {x.name: x for x in manifest.params} return manifest - # map by action name for + # map action to the final version of the manifest self._exposed_interface: Dict[str, CommandManifest] = { - x.action: _process_manifest(x) for x in exposed_interface + x.action: _update_remapped_params(x) for x in exposed_interface } if len(self._exposed_interface) != len(exposed_interface): raise ValueError( From 3151a1897930d6ca5458ed24b11f1a3879c275fd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:28:18 +0200 Subject: [PATCH 55/86] more refactoring --- src/osparc_control/core.py | 31 +++++++++---------- src/osparc_control/errors.py | 2 +- .../transport/base_transport.py | 8 +++++ tests/test_core.py | 8 ++--- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 7779e40..2d5de7a 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -18,7 +18,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .errors import CommnadNotAcceptedError +from .errors import CommandNotAcceptedError from .errors import NoCommandReceivedArrivedError from .errors import NoReplyError from .models import CommandManifest @@ -111,20 +111,17 @@ def __exit__(self, *args: Any) -> None: self.stop_background_sync() def _sender_worker(self) -> None: - self._sender_receiver_pair.sender_init() - - while self._continue: - message: Optional[ - Union[CommandRequest, CommandReceived, CommandReply] - ] = self._out_queue.get() - if message is None: - # exit worker - break - - # send message - self._sender_receiver_pair.send_bytes(message.to_bytes()) - - self._sender_receiver_pair.sender_cleanup() + with self._sender_receiver_pair: + while self._continue: + message: Optional[ + Union[CommandRequest, CommandReceived, CommandReply] + ] = self._out_queue.get() + if message is None: + # exit worker + break + + # send message + self._sender_receiver_pair.send_bytes(message.to_bytes()) def _handle_command_request(self, response: bytes) -> None: command_request: Optional[CommandRequest] = CommandRequest.from_bytes(response) @@ -267,7 +264,7 @@ def _enqueue_call( assert command_received # noqa: S101 if not command_received.accepted: - raise CommnadNotAcceptedError(command_received.error_message) + raise CommandNotAcceptedError(command_received.error_message) return request @@ -365,7 +362,7 @@ def request_with_immediate_reply( def get_incoming_requests(self) -> List[CommandRequest]: """ - Non blocking, retruns all accumulated CommandRequests. + Non blocking, reruns all accumulated CommandRequests. It is meant to be used in an existing cycle """ results: Deque[CommandRequest] = deque() diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index 3f18738..d64afe2 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -6,7 +6,7 @@ class NoReplyError(BaseControlError): """Used when retrying for a result""" -class CommnadNotAcceptedError(BaseControlError): +class CommandNotAcceptedError(BaseControlError): """Command was not accepted by remote""" diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index ae07346..86b5291 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -1,5 +1,6 @@ from abc import ABCMeta from abc import abstractmethod +from typing import Any from typing import Optional @@ -72,3 +73,10 @@ def sender_cleanup(self) -> None: def receiver_cleanup(self) -> None: self._receiver.receiver_cleanup() + + def __enter__(self) -> "SenderReceiverPair": + self.sender_init() + return self + + def __exit__(self, *args: Any) -> None: + self.sender_cleanup() diff --git a/tests/test_core.py b/tests/test_core.py index 390f7b4..2657672 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7,7 +7,7 @@ import osparc_control from osparc_control.core import ControlInterface -from osparc_control.errors import CommnadNotAcceptedError +from osparc_control.errors import CommandNotAcceptedError from osparc_control.errors import NoCommandReceivedArrivedError from osparc_control.models import CommandManifest from osparc_control.models import CommandParameter @@ -195,21 +195,21 @@ def test_no_same_action_command_in_exposed_interface(command_type: CommandType) def test_no_registered_command( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: - with pytest.raises(CommnadNotAcceptedError): + with pytest.raises(CommandNotAcceptedError): control_interface_a.request_without_reply("command_not_defined") def test_wrong_command_type( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: - with pytest.raises(CommnadNotAcceptedError): + with pytest.raises(CommandNotAcceptedError): control_interface_a.request_without_reply("add_numbers") def test_command_params_mismatch( control_interface_a: ControlInterface, control_interface_b: ControlInterface ) -> None: - with pytest.raises(CommnadNotAcceptedError): + with pytest.raises(CommandNotAcceptedError): control_interface_a.request_without_reply("add_numbers", {"nope": 123}) From e5d01e878626e8e7ed2c536aae9ab0bd6902c92e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:33:40 +0200 Subject: [PATCH 56/86] refactor and optimized --- src/osparc_control/core.py | 5 ++--- src/osparc_control/models.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 2d5de7a..49dc478 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -74,7 +74,7 @@ def __init__( ) def _update_remapped_params(manifest: CommandManifest) -> CommandManifest: - manifest._remapped_params = {x.name: x for x in manifest.params} + manifest._params_names_set = {x.name for x in manifest.params} return manifest # map action to the final version of the manifest @@ -159,8 +159,7 @@ def _refuse_and_return(error_message: str) -> None: # check if provided parametes match manifest incoming_params_set = set(command_request.params.keys()) - manifest_params_set = set(manifest._remapped_params.keys()) - if incoming_params_set != manifest_params_set: + if incoming_params_set != manifest._params_names_set: error_message = ( f"Incoming request params {command_request.params} do not match " f"manifest's params {manifest.params}" diff --git a/src/osparc_control/models.py b/src/osparc_control/models.py index 788ee43..c77bf81 100644 --- a/src/osparc_control/models.py +++ b/src/osparc_control/models.py @@ -3,6 +3,7 @@ from typing import Dict from typing import List from typing import Optional +from typing import Set import umsgpack # type: ignore from pydantic import BaseModel @@ -52,8 +53,8 @@ class CommandType(str, Enum): class CommandManifest(BaseModel): - # used internally - _remapped_params: Dict[str, CommandParameter] = PrivateAttr() + # used to speed up parameter matching + _params_names_set: Set[str] = PrivateAttr() action: str = Field(..., description="name of the action to be triggered on remote") description: str = Field(..., description="more details about the action") From 868f572d9a21866be628e10bcda8ebc7505ffd47 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 31 Mar 2022 16:34:11 +0200 Subject: [PATCH 57/86] typo --- src/osparc_control/transport/base_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osparc_control/transport/base_transport.py b/src/osparc_control/transport/base_transport.py index 86b5291..4d52d75 100644 --- a/src/osparc_control/transport/base_transport.py +++ b/src/osparc_control/transport/base_transport.py @@ -47,7 +47,7 @@ def receiver_cleanup(self) -> None: # noqa: N804 class SenderReceiverPair: - """To be used by more custom protcols""" + """To be used by more custom protocols""" def __init__(self, sender: BaseTransport, receiver: BaseTransport) -> None: self._sender: BaseTransport = sender From 5e5adea41f557a6fc3722661e25dfeb93c534480 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 1 Apr 2022 09:33:06 +0200 Subject: [PATCH 58/86] enhanced request uniquness by adding a session id --- src/osparc_control/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 49dc478..aeb4ca4 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -11,6 +11,7 @@ from typing import Tuple from typing import Union from uuid import getnode +from uuid import UUID from uuid import uuid4 from pydantic import ValidationError @@ -42,10 +43,12 @@ DEFAULT_LISTEN_PORT: int = 7426 +UNIQUE_HARDWARE_ID: int = getnode() +SESSION_ID: UUID = uuid4() + def _generate_request_id() -> str: - unique_hardware_id: int = getnode() - return f"{unique_hardware_id}_{uuid4()}" + return f"{UNIQUE_HARDWARE_ID}.{SESSION_ID}_{uuid4()}" def _get_sender_receiver_pair( From 194069851edf2afd24bad891a0f750e60c342520 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 10:15:07 +0200 Subject: [PATCH 59/86] added validation --- examples/base_time_add/sidecar_controller.py | 1 - examples/base_time_add/sidecar_solver.py | 1 - src/osparc_control/core.py | 3 ++- tests/test_core.py | 11 +++++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py index 270c525..8d1d2be 100644 --- a/examples/base_time_add/sidecar_controller.py +++ b/examples/base_time_add/sidecar_controller.py @@ -1,6 +1,5 @@ from osparc_control import ControlInterface - control_interface = ControlInterface( remote_host="localhost", exposed_interface=[], remote_port=1234, listen_port=1235 ) diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py index 7afe005..42e050e 100644 --- a/examples/base_time_add/sidecar_solver.py +++ b/examples/base_time_add/sidecar_solver.py @@ -3,7 +3,6 @@ from osparc_control import CommandType from osparc_control import ControlInterface - command_add = CommandManifest( action="add_internal_time", description="adds internal time to a provided paramter", diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index aeb4ca4..1e439a8 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -14,6 +14,7 @@ from uuid import UUID from uuid import uuid4 +from pydantic import validate_arguments from pydantic import ValidationError from tenacity import Retrying from tenacity.stop import stop_after_delay @@ -32,7 +33,6 @@ from .transport.base_transport import SenderReceiverPair from osparc_control.transport.zeromq import ZeroMQTransport - _MINUTE: float = 60.0 WAIT_FOR_MESSAGES: float = 0.01 @@ -64,6 +64,7 @@ def _get_sender_receiver_pair( class ControlInterface: + @validate_arguments def __init__( self, remote_host: str, diff --git a/tests/test_core.py b/tests/test_core.py index 2657672..345cdc8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,6 +4,7 @@ from typing import List import pytest +from pydantic import ValidationError import osparc_control from osparc_control.core import ControlInterface @@ -220,3 +221,13 @@ def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: control_interface.request_without_reply( "no_remote_side_for_command", {"nope": 123} ) + + +def test_control_interface_validation() -> None: + with pytest.raises(ValidationError): + ControlInterface( + remote_host="localhost", + exposed_interface=[1], + remote_port=1, + listen_port=2, + ) From 2b6cbd32dd6a56107675187335a7c5b593fb8f3b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 10:25:22 +0200 Subject: [PATCH 60/86] improve parsing of commands --- src/osparc_control/core.py | 9 ++++++--- src/osparc_control/transport/zeromq.py | 11 ++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 1e439a8..a64a01f 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -200,7 +200,8 @@ def _receiver_worker(self) -> None: self._sender_receiver_pair.receiver_init() while self._continue: - # this is blocking should be under timeout block + sleep(WAIT_FOR_MESSAGES) + response: Optional[bytes] = self._sender_receiver_pair.receive_bytes() if response is None: # no messages available @@ -208,27 +209,29 @@ def _receiver_worker(self) -> None: # NOTE: pydantic does not support polymorphism # SEE https://github.com/samuelcolvin/pydantic/issues/503 + # below try catch pattern is how to deal with it # case CommandReceived try: self._handle_command_received(response) + continue except ValidationError: pass # case CommandRequest try: self._handle_command_request(response) + continue except ValidationError: pass # case CommandReply try: self._handle_command_reply(response) + continue except ValidationError: pass - sleep(WAIT_FOR_MESSAGES) - self._sender_receiver_pair.receiver_cleanup() def _enqueue_call( diff --git a/src/osparc_control/transport/zeromq.py b/src/osparc_control/transport/zeromq.py index d20d888..4554107 100644 --- a/src/osparc_control/transport/zeromq.py +++ b/src/osparc_control/transport/zeromq.py @@ -10,6 +10,9 @@ from .base_transport import BaseTransport +RETRY_COUNT: int = 3 +WAIT_BETWEEN: float = 0.01 + class ZeroMQTransport(BaseTransport): def __init__(self, listen_port: int, remote_host: str, remote_port: int): @@ -27,18 +30,16 @@ def send_bytes(self, payload: bytes) -> None: self._send_socket.send(payload) # type: ignore - def receive_bytes( - self, retry_count: int = 3, wait_between: float = 0.01 - ) -> Optional[bytes]: + def receive_bytes(self) -> Optional[bytes]: assert self._recv_socket # noqa: S101 - # try to fetch a message, usning unlocking sockets does not guarantee + # try to fetch a message, using blocking sockets does not guarantee # that data is always present, retry 3 times in a short amount of time # this will guarantee the message arrives message: Optional[bytes] = None try: for attempt in Retrying( - stop=stop_after_attempt(retry_count), wait=wait_fixed(wait_between) + stop=stop_after_attempt(RETRY_COUNT), wait=wait_fixed(WAIT_BETWEEN) ): with attempt: message = self._recv_socket.recv(zmq.NOBLOCK) # type: ignore From e30613d8fb1849827f120f2e375493f8dc2a30f3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 10:31:35 +0200 Subject: [PATCH 61/86] improvde comments --- src/osparc_control/core.py | 15 ++++++++++----- src/osparc_control/errors.py | 2 +- tests/test_core.py | 10 +++++----- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index a64a01f..64ead16 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -20,8 +20,8 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +from .errors import CommandConfirmationTimeoutError from .errors import CommandNotAcceptedError -from .errors import NoCommandReceivedArrivedError from .errors import NoReplyError from .models import CommandManifest from .models import CommandReceived @@ -39,7 +39,7 @@ WAIT_BETWEEN_CHECKS: float = 0.1 # NOTE: this effectively limits the time between when # the two remote sides can start to communicate -WAIT_FOR_RECEIVED: float = 1 * _MINUTE +WAIT_FOR_RECEIVED_S: float = 1 * _MINUTE DEFAULT_LISTEN_PORT: int = 7426 @@ -254,18 +254,23 @@ def _enqueue_call( self._out_queue.put(request) - # check command_received status + # wait for remote to reply with command_received + # if no reply is received in WAIT_FOR_RECEIVED_S + # an error will be raised + # an error will also be raised if the command was + # unexpected (did not validate agains a + # CommandManifest entry) command_received: Optional[CommandReceived] = None try: for attempt in Retrying( - stop=stop_after_delay(WAIT_FOR_RECEIVED), + stop=stop_after_delay(WAIT_FOR_RECEIVED_S), wait=wait_fixed(WAIT_BETWEEN_CHECKS), retry_error_cls=Empty, # type: ignore ): with attempt: command_received = self._incoming_command_queue.get(block=False) except Empty: - raise NoCommandReceivedArrivedError() from None + raise CommandConfirmationTimeoutError() from None assert command_received # noqa: S101 diff --git a/src/osparc_control/errors.py b/src/osparc_control/errors.py index d64afe2..1a4dd77 100644 --- a/src/osparc_control/errors.py +++ b/src/osparc_control/errors.py @@ -10,5 +10,5 @@ class CommandNotAcceptedError(BaseControlError): """Command was not accepted by remote""" -class NoCommandReceivedArrivedError(BaseControlError): +class CommandConfirmationTimeoutError(BaseControlError): """Reply from remote host did not arrive in time""" diff --git a/tests/test_core.py b/tests/test_core.py index 345cdc8..0d8b5fe 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,8 +8,8 @@ import osparc_control from osparc_control.core import ControlInterface +from osparc_control.errors import CommandConfirmationTimeoutError from osparc_control.errors import CommandNotAcceptedError -from osparc_control.errors import NoCommandReceivedArrivedError from osparc_control.models import CommandManifest from osparc_control.models import CommandParameter from osparc_control.models import CommandType @@ -88,10 +88,10 @@ def control_interface_b( @pytest.fixture def mock_wait_for_received() -> Iterable[None]: - previous = osparc_control.core.WAIT_FOR_RECEIVED - osparc_control.core.WAIT_FOR_RECEIVED = 0.01 + previous = osparc_control.core.WAIT_FOR_RECEIVED_S + osparc_control.core.WAIT_FOR_RECEIVED_S = 0.01 yield - osparc_control.core.WAIT_FOR_RECEIVED = previous + osparc_control.core.WAIT_FOR_RECEIVED_S = previous # TESTS @@ -217,7 +217,7 @@ def test_command_params_mismatch( def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: control_interface = _get_control_interface(8263, 8263, []) control_interface.start_background_sync() - with pytest.raises(NoCommandReceivedArrivedError): + with pytest.raises(CommandConfirmationTimeoutError): control_interface.request_without_reply( "no_remote_side_for_command", {"nope": 123} ) From 5b2bab01d9add4b8fc0e6ca7e72a144e85b3a13e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 10:39:02 +0200 Subject: [PATCH 62/86] renamed ControlInterface to PairedTransmitter --- docs/reference.rst | 2 +- examples/base_time_add/sidecar_controller.py | 18 ++--- examples/base_time_add/sidecar_solver.py | 4 +- examples/base_time_add/time_solver.py | 20 ++--- examples/basic/replyer.py | 12 +-- examples/basic/requester.py | 10 +-- src/osparc_control/__init__.py | 4 +- src/osparc_control/core.py | 4 +- tests/test_core.py | 84 ++++++++++---------- 9 files changed, 80 insertions(+), 78 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 6694704..f75998d 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -5,7 +5,7 @@ Reference osparc_control -------------- -.. autoclass:: osparc_control.ControlInterface +.. autoclass:: osparc_control.PairedTransmitter :members: :undoc-members: diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py index 8d1d2be..aea850e 100644 --- a/examples/base_time_add/sidecar_controller.py +++ b/examples/base_time_add/sidecar_controller.py @@ -1,36 +1,36 @@ -from osparc_control import ControlInterface +from osparc_control import PairedTransmitter -control_interface = ControlInterface( +paired_transmitter = PairedTransmitter( remote_host="localhost", exposed_interface=[], remote_port=1234, listen_port=1235 ) -control_interface.start_background_sync() +paired_transmitter.start_background_sync() # add_internal_time add_params = {"a": 10} print("Will add ", add_params) -request_id = control_interface.request_with_delayed_reply( +request_id = paired_transmitter.request_with_delayed_reply( "add_internal_time", params=add_params ) has_result = False result = None while not has_result: - has_result, result = control_interface.check_for_reply(request_id=request_id) + has_result, result = paired_transmitter.check_for_reply(request_id=request_id) print("result of addition", result) # get_time print("getting solver time") -solver_time = control_interface.request_with_immediate_reply("get_time", timeout=1.0) -random_int = control_interface.request_with_immediate_reply( +solver_time = paired_transmitter.request_with_immediate_reply("get_time", timeout=1.0) +random_int = paired_transmitter.request_with_immediate_reply( "random_in_range", timeout=1.0, params={"a": 1, "b": 3} ) print("solver time", solver_time) print("sending command to print internal status") -control_interface.request_without_reply("print_status") +paired_transmitter.request_without_reply("print_status") -control_interface.stop_background_sync() +paired_transmitter.stop_background_sync() diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py index 42e050e..df8e4fa 100644 --- a/examples/base_time_add/sidecar_solver.py +++ b/examples/base_time_add/sidecar_solver.py @@ -1,7 +1,7 @@ from osparc_control import CommandManifest from osparc_control import CommandParameter from osparc_control import CommandType -from osparc_control import ControlInterface +from osparc_control import PairedTransmitter command_add = CommandManifest( action="add_internal_time", @@ -27,7 +27,7 @@ ) -control_interface = ControlInterface( +paired_transmitter = PairedTransmitter( remote_host="localhost", exposed_interface=[command_add, command_get_time, command_print_solver_status], remote_port=1235, diff --git a/examples/base_time_add/time_solver.py b/examples/base_time_add/time_solver.py index d341fcc..0cb780d 100644 --- a/examples/base_time_add/time_solver.py +++ b/examples/base_time_add/time_solver.py @@ -3,22 +3,22 @@ from sidecar_solver import command_add from sidecar_solver import command_get_time from sidecar_solver import command_print_solver_status -from sidecar_solver import control_interface +from sidecar_solver import paired_transmitter -from osparc_control.core import ControlInterface +from osparc_control.core import PairedTransmitter from osparc_control.models import CommandRequest def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: if request.action == command_add.action: sum_result = time_solver._add(**request.params) - time_solver.control_interface.reply_to_command( + time_solver.paired_transmitter.reply_to_command( request_id=request.request_id, payload=sum_result ) return if request.action == command_get_time.action: - time_solver.control_interface.reply_to_command( + time_solver.paired_transmitter.reply_to_command( request_id=request.request_id, payload=time_solver.time ) return @@ -32,10 +32,10 @@ def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: class TimeSolver: def __init__( - self, initial_time: float, control_interface: ControlInterface + self, initial_time: float, paired_transmitter: PairedTransmitter ) -> None: self.time = initial_time - self.control_interface = control_interface + self.paired_transmitter = paired_transmitter # internal time tick self.sleep_internal: float = 0.1 @@ -54,7 +54,7 @@ def run(self) -> None: """main loop of the solver""" while self.can_continue: # process incoming requests from remote - for request in self.control_interface.get_incoming_requests(): + for request in self.paired_transmitter.get_incoming_requests(): handle_inputs(time_solver=self, request=request) # process internal stuff @@ -63,12 +63,12 @@ def run(self) -> None: def main() -> None: - control_interface.start_background_sync() + paired_transmitter.start_background_sync() - solver = TimeSolver(initial_time=0, control_interface=control_interface) + solver = TimeSolver(initial_time=0, paired_transmitter=paired_transmitter) solver.run() - control_interface.stop_background_sync() + paired_transmitter.stop_background_sync() if __name__ == "__main__": diff --git a/examples/basic/replyer.py b/examples/basic/replyer.py index 5226bd4..85fd40e 100644 --- a/examples/basic/replyer.py +++ b/examples/basic/replyer.py @@ -4,7 +4,7 @@ from osparc_control import CommandManifest from osparc_control import CommandParameter from osparc_control import CommandType -from osparc_control import ControlInterface +from osparc_control import PairedTransmitter random_in_range_manifest = CommandManifest( action="random_in_range", @@ -16,22 +16,22 @@ command_type=CommandType.WITH_IMMEDIATE_REPLY, ) -control_interface = ControlInterface( +paired_transmitter = PairedTransmitter( remote_host="localhost", exposed_interface=[random_in_range_manifest], remote_port=2346, listen_port=2345, ) -control_interface.start_background_sync() +paired_transmitter.start_background_sync() wait_for_requests = True while wait_for_requests: - for command in control_interface.get_incoming_requests(): + for command in paired_transmitter.get_incoming_requests(): if command.action == random_in_range_manifest.action: random_int = random.randint( # noqa: S311 command.params["a"], command.params["b"] ) - control_interface.reply_to_command( + paired_transmitter.reply_to_command( request_id=command.request_id, payload=random_int ) wait_for_requests = False @@ -39,4 +39,4 @@ # allow for message to be delivered time.sleep(0.01) -control_interface.stop_background_sync() +paired_transmitter.stop_background_sync() diff --git a/examples/basic/requester.py b/examples/basic/requester.py index fa34c79..d475e7a 100644 --- a/examples/basic/requester.py +++ b/examples/basic/requester.py @@ -1,15 +1,15 @@ -from osparc_control import ControlInterface +from osparc_control import PairedTransmitter -control_interface = ControlInterface( +paired_transmitter = PairedTransmitter( remote_host="localhost", exposed_interface=[], remote_port=2345, listen_port=2346 ) -control_interface.start_background_sync() +paired_transmitter.start_background_sync() -random_int = control_interface.request_with_immediate_reply( +random_int = paired_transmitter.request_with_immediate_reply( "random_in_range", timeout=10.0, params={"a": 1, "b": 10} ) print(random_int) assert 1 <= random_int <= 10 # noqa: S101 -control_interface.stop_background_sync() +paired_transmitter.stop_background_sync() diff --git a/src/osparc_control/__init__.py b/src/osparc_control/__init__.py index a810ad0..869502f 100644 --- a/src/osparc_control/__init__.py +++ b/src/osparc_control/__init__.py @@ -1,7 +1,7 @@ """Osparc Control.""" -from .core import ControlInterface +from .core import PairedTransmitter from .models import CommandManifest from .models import CommandParameter from .models import CommandType -__all__ = ["ControlInterface", "CommandManifest", "CommandParameter", "CommandType"] +__all__ = ["PairedTransmitter", "CommandManifest", "CommandParameter", "CommandType"] diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 64ead16..d8de7d7 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -63,7 +63,7 @@ def _get_sender_receiver_pair( return SenderReceiverPair(sender=sender, receiver=receiver) -class ControlInterface: +class PairedTransmitter: @validate_arguments def __init__( self, @@ -107,7 +107,7 @@ def _update_remapped_params(manifest: CommandManifest) -> CommandManifest: ) self._continue: bool = True - def __enter__(self) -> "ControlInterface": + def __enter__(self) -> "PairedTransmitter": self.start_background_sync() return self diff --git a/tests/test_core.py b/tests/test_core.py index 0d8b5fe..4f2d125 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7,7 +7,7 @@ from pydantic import ValidationError import osparc_control -from osparc_control.core import ControlInterface +from osparc_control.core import PairedTransmitter from osparc_control.errors import CommandConfirmationTimeoutError from osparc_control.errors import CommandNotAcceptedError from osparc_control.models import CommandManifest @@ -25,10 +25,10 @@ # UTILS -def _get_control_interface( +def _get_paired_transmitter( local_port: int, remote_port: int, exposed_interface: List[CommandManifest] -) -> ControlInterface: - return ControlInterface( +) -> PairedTransmitter: + return PairedTransmitter( remote_host="localhost", exposed_interface=exposed_interface, remote_port=remote_port, @@ -40,11 +40,11 @@ def _get_control_interface( @pytest.fixture -def control_interface_a() -> Iterable[ControlInterface]: - control_interface = _get_control_interface(1234, 1235, []) - control_interface.start_background_sync() - yield control_interface - control_interface.stop_background_sync() +def paired_transmitter_a() -> Iterable[PairedTransmitter]: + paired_transmitter = _get_paired_transmitter(1234, 1235, []) + paired_transmitter.start_background_sync() + yield paired_transmitter + paired_transmitter.stop_background_sync() @pytest.fixture @@ -77,13 +77,13 @@ def mainfest_b() -> List[CommandManifest]: @pytest.fixture -def control_interface_b( +def paired_transmitter_b( mainfest_b: List[CommandManifest], -) -> Iterable[ControlInterface]: - control_interface = _get_control_interface(1235, 1234, mainfest_b) - control_interface.start_background_sync() - yield control_interface - control_interface.stop_background_sync() +) -> Iterable[PairedTransmitter]: + paired_transmitter = _get_paired_transmitter(1235, 1234, mainfest_b) + paired_transmitter.start_background_sync() + yield paired_transmitter + paired_transmitter.stop_background_sync() @pytest.fixture @@ -98,24 +98,24 @@ def mock_wait_for_received() -> Iterable[None]: def test_context_manager(mainfest_b: List[CommandManifest]) -> None: - with _get_control_interface(1235, 1234, mainfest_b): + with _get_paired_transmitter(1235, 1234, mainfest_b): pass def test_request_with_delayed_reply( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: # SIDE A - request_id = control_interface_a.request_with_delayed_reply( + request_id = paired_transmitter_a.request_with_delayed_reply( "add_numbers", params={"a": 10, "b": 13.3} ) # SIDE B wait_for_requests = True while wait_for_requests: - for command in control_interface_b.get_incoming_requests(): + for command in paired_transmitter_b.get_incoming_requests(): assert command.action == "add_numbers" - control_interface_b.reply_to_command( + paired_transmitter_b.reply_to_command( request_id=command.request_id, payload=sum(command.params.values()) ) wait_for_requests = False @@ -123,21 +123,21 @@ def test_request_with_delayed_reply( # SIDE A time.sleep(WAIT_FOR_DELIVERY) - has_result, result = control_interface_a.check_for_reply(request_id=request_id) + has_result, result = paired_transmitter_a.check_for_reply(request_id=request_id) assert has_result is True assert result is not None def test_request_with_immediate_reply( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: def _worker_b() -> None: count = 1 wait_for_requests = True while wait_for_requests: - for command in control_interface_b.get_incoming_requests(): + for command in paired_transmitter_b.get_incoming_requests(): assert command.action == "get_random" - control_interface_b.reply_to_command( + paired_transmitter_b.reply_to_command( request_id=command.request_id, payload=random.randint(1, 1000), # noqa: S311 ) @@ -150,14 +150,14 @@ def _worker_b() -> None: thread = Thread(target=_worker_b, daemon=True) thread.start() - random_integer = control_interface_a.request_with_immediate_reply( + random_integer = paired_transmitter_a.request_with_immediate_reply( "get_random", timeout=1.0 ) assert type(random_integer) == int assert random_integer assert 1 <= random_integer <= 1000 - no_reply_in_time = control_interface_a.request_with_immediate_reply( + no_reply_in_time = paired_transmitter_a.request_with_immediate_reply( "get_random", timeout=0.001 ) assert no_reply_in_time is None @@ -166,17 +166,17 @@ def _worker_b() -> None: def test_request_without_reply( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: # SIDE A - control_interface_a.request_without_reply("greet_user", params={"name": "tester"}) + paired_transmitter_a.request_without_reply("greet_user", params={"name": "tester"}) expected_message = "hello tester" # SIDE B wait_for_requests = True while wait_for_requests: - for command in control_interface_b.get_incoming_requests(): + for command in paired_transmitter_b.get_incoming_requests(): assert command.action == "greet_user" message = f"hello {command.params['name']}" assert message == expected_message @@ -190,42 +190,44 @@ def test_no_same_action_command_in_exposed_interface(command_type: CommandType) ) with pytest.raises(ValueError): - _get_control_interface(100, 100, [test_command_manifest, test_command_manifest]) + _get_paired_transmitter( + 100, 100, [test_command_manifest, test_command_manifest] + ) def test_no_registered_command( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: with pytest.raises(CommandNotAcceptedError): - control_interface_a.request_without_reply("command_not_defined") + paired_transmitter_a.request_without_reply("command_not_defined") def test_wrong_command_type( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: with pytest.raises(CommandNotAcceptedError): - control_interface_a.request_without_reply("add_numbers") + paired_transmitter_a.request_without_reply("add_numbers") def test_command_params_mismatch( - control_interface_a: ControlInterface, control_interface_b: ControlInterface + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter ) -> None: with pytest.raises(CommandNotAcceptedError): - control_interface_a.request_without_reply("add_numbers", {"nope": 123}) + paired_transmitter_a.request_without_reply("add_numbers", {"nope": 123}) def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: - control_interface = _get_control_interface(8263, 8263, []) - control_interface.start_background_sync() + paired_transmitter = _get_paired_transmitter(8263, 8263, []) + paired_transmitter.start_background_sync() with pytest.raises(CommandConfirmationTimeoutError): - control_interface.request_without_reply( + paired_transmitter.request_without_reply( "no_remote_side_for_command", {"nope": 123} ) -def test_control_interface_validation() -> None: +def test_paired_transmitter_validation() -> None: with pytest.raises(ValidationError): - ControlInterface( + PairedTransmitter( remote_host="localhost", exposed_interface=[1], remote_port=1, From cf911c7f3da39a3dd2d3356885d1cb4c9872e195 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 10:45:18 +0200 Subject: [PATCH 63/86] renamed exposed_interface to exposed_commands --- examples/base_time_add/sidecar_controller.py | 2 +- examples/base_time_add/sidecar_solver.py | 2 +- examples/basic/replyer.py | 2 +- examples/basic/requester.py | 2 +- src/osparc_control/core.py | 17 +++++++++-------- tests/test_core.py | 8 ++++---- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py index aea850e..77987a2 100644 --- a/examples/base_time_add/sidecar_controller.py +++ b/examples/base_time_add/sidecar_controller.py @@ -1,7 +1,7 @@ from osparc_control import PairedTransmitter paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_interface=[], remote_port=1234, listen_port=1235 + remote_host="localhost", exposed_commands=[], remote_port=1234, listen_port=1235 ) paired_transmitter.start_background_sync() diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py index df8e4fa..c7b09fb 100644 --- a/examples/base_time_add/sidecar_solver.py +++ b/examples/base_time_add/sidecar_solver.py @@ -29,7 +29,7 @@ paired_transmitter = PairedTransmitter( remote_host="localhost", - exposed_interface=[command_add, command_get_time, command_print_solver_status], + exposed_commands=[command_add, command_get_time, command_print_solver_status], remote_port=1235, listen_port=1234, ) diff --git a/examples/basic/replyer.py b/examples/basic/replyer.py index 85fd40e..0fccaf3 100644 --- a/examples/basic/replyer.py +++ b/examples/basic/replyer.py @@ -18,7 +18,7 @@ paired_transmitter = PairedTransmitter( remote_host="localhost", - exposed_interface=[random_in_range_manifest], + exposed_commands=[random_in_range_manifest], remote_port=2346, listen_port=2345, ) diff --git a/examples/basic/requester.py b/examples/basic/requester.py index d475e7a..c665716 100644 --- a/examples/basic/requester.py +++ b/examples/basic/requester.py @@ -1,7 +1,7 @@ from osparc_control import PairedTransmitter paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_interface=[], remote_port=2345, listen_port=2346 + remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 ) paired_transmitter.start_background_sync() diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index d8de7d7..40c818b 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -68,7 +68,8 @@ class PairedTransmitter: def __init__( self, remote_host: str, - exposed_interface: List[CommandManifest], + *, + exposed_commands: List[CommandManifest], remote_port: int = DEFAULT_LISTEN_PORT, listen_port: int = DEFAULT_LISTEN_PORT, ) -> None: @@ -82,12 +83,12 @@ def _update_remapped_params(manifest: CommandManifest) -> CommandManifest: return manifest # map action to the final version of the manifest - self._exposed_interface: Dict[str, CommandManifest] = { - x.action: _update_remapped_params(x) for x in exposed_interface + self._exposed_commands: Dict[str, CommandManifest] = { + x.action: _update_remapped_params(x) for x in exposed_commands } - if len(self._exposed_interface) != len(exposed_interface): + if len(self._exposed_commands) != len(exposed_commands): raise ValueError( - f"Provided exposed_interface={exposed_interface} " + f"Provided exposed_commands={exposed_commands} " "contains CommandManifest with same action name." ) @@ -143,14 +144,14 @@ def _refuse_and_return(error_message: str) -> None: return # check if command exists - if command_request.action not in self._exposed_interface: + if command_request.action not in self._exposed_commands: error_message = ( f"No registered command found for action={command_request.action}. " - f"Supported actions {list(self._exposed_interface.keys())}" + f"Supported actions {list(self._exposed_commands.keys())}" ) _refuse_and_return(error_message) - manifest = self._exposed_interface[command_request.action] + manifest = self._exposed_commands[command_request.action] # check command_type matches the one declared in the manifest if command_request.command_type != manifest.command_type: diff --git a/tests/test_core.py b/tests/test_core.py index 4f2d125..990c74e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -26,11 +26,11 @@ def _get_paired_transmitter( - local_port: int, remote_port: int, exposed_interface: List[CommandManifest] + local_port: int, remote_port: int, exposed_commands: List[CommandManifest] ) -> PairedTransmitter: return PairedTransmitter( remote_host="localhost", - exposed_interface=exposed_interface, + exposed_commands=exposed_commands, remote_port=remote_port, listen_port=local_port, ) @@ -184,7 +184,7 @@ def test_request_without_reply( @pytest.mark.parametrize("command_type", ALL_COMMAND_TYPES) -def test_no_same_action_command_in_exposed_interface(command_type: CommandType) -> None: +def test_no_same_action_command_in_exposed_commands(command_type: CommandType) -> None: test_command_manifest = CommandManifest( action="test", description="test", params=[], command_type=command_type ) @@ -229,7 +229,7 @@ def test_paired_transmitter_validation() -> None: with pytest.raises(ValidationError): PairedTransmitter( remote_host="localhost", - exposed_interface=[1], + exposed_commands=[1], # type: ignore remote_port=1, listen_port=2, ) From adf3fd54bbb96c4ec927eacbd0bfa19123d5ac22 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 11:00:12 +0200 Subject: [PATCH 64/86] making tests more readbale --- tests/test_core.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 990c74e..8e1a2e5 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -26,7 +26,7 @@ def _get_paired_transmitter( - local_port: int, remote_port: int, exposed_commands: List[CommandManifest] + *, local_port: int, remote_port: int, exposed_commands: List[CommandManifest] ) -> PairedTransmitter: return PairedTransmitter( remote_host="localhost", @@ -41,7 +41,9 @@ def _get_paired_transmitter( @pytest.fixture def paired_transmitter_a() -> Iterable[PairedTransmitter]: - paired_transmitter = _get_paired_transmitter(1234, 1235, []) + paired_transmitter = _get_paired_transmitter( + local_port=1234, remote_port=1235, exposed_commands=[] + ) paired_transmitter.start_background_sync() yield paired_transmitter paired_transmitter.stop_background_sync() @@ -80,7 +82,9 @@ def mainfest_b() -> List[CommandManifest]: def paired_transmitter_b( mainfest_b: List[CommandManifest], ) -> Iterable[PairedTransmitter]: - paired_transmitter = _get_paired_transmitter(1235, 1234, mainfest_b) + paired_transmitter = _get_paired_transmitter( + local_port=1235, remote_port=1234, exposed_commands=mainfest_b + ) paired_transmitter.start_background_sync() yield paired_transmitter paired_transmitter.stop_background_sync() @@ -98,7 +102,9 @@ def mock_wait_for_received() -> Iterable[None]: def test_context_manager(mainfest_b: List[CommandManifest]) -> None: - with _get_paired_transmitter(1235, 1234, mainfest_b): + with _get_paired_transmitter( + local_port=1235, remote_port=1234, exposed_commands=mainfest_b + ): pass @@ -191,7 +197,9 @@ def test_no_same_action_command_in_exposed_commands(command_type: CommandType) - with pytest.raises(ValueError): _get_paired_transmitter( - 100, 100, [test_command_manifest, test_command_manifest] + local_port=100, + remote_port=100, + exposed_commands=[test_command_manifest, test_command_manifest], ) @@ -217,7 +225,9 @@ def test_command_params_mismatch( def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: - paired_transmitter = _get_paired_transmitter(8263, 8263, []) + paired_transmitter = _get_paired_transmitter( + local_port=8263, remote_port=8263, exposed_commands=[] + ) paired_transmitter.start_background_sync() with pytest.raises(CommandConfirmationTimeoutError): paired_transmitter.request_without_reply( From c01d89deeab4ba2d3210f5f687027fe9d2aeff84 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 14:51:58 +0200 Subject: [PATCH 65/86] timeout has to be explicit when calling fucntiuon --- src/osparc_control/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osparc_control/core.py b/src/osparc_control/core.py index 40c818b..e4bbb86 100644 --- a/src/osparc_control/core.py +++ b/src/osparc_control/core.py @@ -345,8 +345,9 @@ def check_for_reply(self, request_id: str) -> Tuple[bool, Optional[Any]]: def request_with_immediate_reply( self, action: str, - timeout: float, params: Optional[Dict[str, Any]] = None, + *, + timeout: float, ) -> Optional[Any]: """ Requests and awaits for the response from remote. From 586020a659ee156ff56a1642790ab3a330275c42 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 15:01:42 +0200 Subject: [PATCH 66/86] refactored the simple example --- examples/basic/replyer.py | 42 ------------------------------------- examples/basic/requester.py | 15 ------------- 2 files changed, 57 deletions(-) delete mode 100644 examples/basic/replyer.py delete mode 100644 examples/basic/requester.py diff --git a/examples/basic/replyer.py b/examples/basic/replyer.py deleted file mode 100644 index 0fccaf3..0000000 --- a/examples/basic/replyer.py +++ /dev/null @@ -1,42 +0,0 @@ -import random -import time - -from osparc_control import CommandManifest -from osparc_control import CommandParameter -from osparc_control import CommandType -from osparc_control import PairedTransmitter - -random_in_range_manifest = CommandManifest( - action="random_in_range", - description="gets the time", - params=[ - CommandParameter(name="a", description="lower bound for random numbers"), - CommandParameter(name="b", description="upper bound for random numbers"), - ], - command_type=CommandType.WITH_IMMEDIATE_REPLY, -) - -paired_transmitter = PairedTransmitter( - remote_host="localhost", - exposed_commands=[random_in_range_manifest], - remote_port=2346, - listen_port=2345, -) -paired_transmitter.start_background_sync() - -wait_for_requests = True -while wait_for_requests: - for command in paired_transmitter.get_incoming_requests(): - if command.action == random_in_range_manifest.action: - random_int = random.randint( # noqa: S311 - command.params["a"], command.params["b"] - ) - paired_transmitter.reply_to_command( - request_id=command.request_id, payload=random_int - ) - wait_for_requests = False - -# allow for message to be delivered -time.sleep(0.01) - -paired_transmitter.stop_background_sync() diff --git a/examples/basic/requester.py b/examples/basic/requester.py deleted file mode 100644 index c665716..0000000 --- a/examples/basic/requester.py +++ /dev/null @@ -1,15 +0,0 @@ -from osparc_control import PairedTransmitter - -paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 -) - -paired_transmitter.start_background_sync() - -random_int = paired_transmitter.request_with_immediate_reply( - "random_in_range", timeout=10.0, params={"a": 1, "b": 10} -) -print(random_int) -assert 1 <= random_int <= 10 # noqa: S101 - -paired_transmitter.stop_background_sync() From 1af657ca2527973fbbeed67b7a1036c7219896b7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 15:01:53 +0200 Subject: [PATCH 67/86] refactored the simple example --- examples/simple/replyer.py | 44 ++++++++++++++++++++++++++++++++++++ examples/simple/requester.py | 15 ++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 examples/simple/replyer.py create mode 100644 examples/simple/requester.py diff --git a/examples/simple/replyer.py b/examples/simple/replyer.py new file mode 100644 index 0000000..fe97b0d --- /dev/null +++ b/examples/simple/replyer.py @@ -0,0 +1,44 @@ +import random +import time + +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommandType +from osparc_control import PairedTransmitter + + +# declare some commands to which a reply can be provided +random_in_range_manifest = CommandManifest( + action="random_in_range", + description="gets the time", + params=[ + CommandParameter(name="a", description="lower bound for random numbers"), + CommandParameter(name="b", description="upper bound for random numbers"), + ], + command_type=CommandType.WITH_IMMEDIATE_REPLY, +) + +paired_transmitter = PairedTransmitter( + remote_host="localhost", + exposed_commands=[random_in_range_manifest], + remote_port=2346, + listen_port=2345, +) +paired_transmitter.start_background_sync() + +wait_for_requests = True +while wait_for_requests: + for command in paired_transmitter.get_incoming_requests(): + if command.action == random_in_range_manifest.action: + random_int = random.randint( # noqa: S311 + command.params["a"], command.params["b"] + ) + paired_transmitter.reply_to_command( + request_id=command.request_id, payload=random_int + ) + wait_for_requests = False + +# allow for message to be delivered +time.sleep(0.01) + +paired_transmitter.stop_background_sync() diff --git a/examples/simple/requester.py b/examples/simple/requester.py new file mode 100644 index 0000000..bf542f4 --- /dev/null +++ b/examples/simple/requester.py @@ -0,0 +1,15 @@ +from osparc_control import PairedTransmitter + +paired_transmitter = PairedTransmitter( + remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 +) + +# using a context manager allows to avoid calling +# paired_transmitter.start_background_sync() before making/receiving requests +# paired_transmitter.stop_background_sync() to close and cleanup when done +with paired_transmitter: + random_int = paired_transmitter.request_with_immediate_reply( + "random_in_range", timeout=10.0, params={"a": 1, "b": 10} + ) + print(random_int) + assert 1 <= random_int <= 10 # noqa: S101 From 4c07c16076b5dca5c402891a0ea0d42484c82687 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 15:53:15 +0200 Subject: [PATCH 68/86] updated examples --- docs/usage.rst | 8 +-- examples/base_time_add/README.md | 8 --- examples/base_time_add/sidecar_controller.py | 36 ---------- examples/base_time_add/sidecar_solver.py | 35 --------- examples/base_time_add/time_solver.py | 75 -------------------- examples/simple/replyer.py | 44 ------------ examples/simple/requester.py | 15 ---- src/osparc_control/__init__.py | 9 ++- 8 files changed, 12 insertions(+), 218 deletions(-) delete mode 100644 examples/base_time_add/README.md delete mode 100644 examples/base_time_add/sidecar_controller.py delete mode 100644 examples/base_time_add/sidecar_solver.py delete mode 100644 examples/base_time_add/time_solver.py delete mode 100644 examples/simple/replyer.py delete mode 100644 examples/simple/requester.py diff --git a/docs/usage.rst b/docs/usage.rst index b088d80..322a3f2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -7,14 +7,14 @@ Please have a look at the examples folder. Minimual examples ================= -In one terminal run ``python examples/basic/requester.py``: +In one terminal run ``python examples/1_simple/requester.py``: -.. literalinclude:: ../examples/basic/requester.py +.. literalinclude:: ../examples/1_simple/requester.py :language: python -In a second terminal run ``python examples/basic/replyer.py``: +In a second terminal run ``python examples/1_simple/replyer.py``: -.. literalinclude:: ../examples/basic/replyer.py +.. literalinclude:: ../examples/1_simple/replyer.py :language: python diff --git a/examples/base_time_add/README.md b/examples/base_time_add/README.md deleted file mode 100644 index 98ff72a..0000000 --- a/examples/base_time_add/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# About - -This example consists of a `time_solver`. Which can add, the current time by a provided value. - -# Usage - -In one terminal run `sidecar_controller.py`. -In a second terminal run `time_solver.py`. It will load data from the `sidecar_solver.py` to use when communicating with the `sidecar_controller.py` diff --git a/examples/base_time_add/sidecar_controller.py b/examples/base_time_add/sidecar_controller.py deleted file mode 100644 index 77987a2..0000000 --- a/examples/base_time_add/sidecar_controller.py +++ /dev/null @@ -1,36 +0,0 @@ -from osparc_control import PairedTransmitter - -paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_commands=[], remote_port=1234, listen_port=1235 -) -paired_transmitter.start_background_sync() - -# add_internal_time - -add_params = {"a": 10} -print("Will add ", add_params) -request_id = paired_transmitter.request_with_delayed_reply( - "add_internal_time", params=add_params -) - -has_result = False -result = None -while not has_result: - has_result, result = paired_transmitter.check_for_reply(request_id=request_id) - -print("result of addition", result) - -# get_time - -print("getting solver time") -solver_time = paired_transmitter.request_with_immediate_reply("get_time", timeout=1.0) -random_int = paired_transmitter.request_with_immediate_reply( - "random_in_range", timeout=1.0, params={"a": 1, "b": 3} -) -print("solver time", solver_time) - -print("sending command to print internal status") -paired_transmitter.request_without_reply("print_status") - - -paired_transmitter.stop_background_sync() diff --git a/examples/base_time_add/sidecar_solver.py b/examples/base_time_add/sidecar_solver.py deleted file mode 100644 index c7b09fb..0000000 --- a/examples/base_time_add/sidecar_solver.py +++ /dev/null @@ -1,35 +0,0 @@ -from osparc_control import CommandManifest -from osparc_control import CommandParameter -from osparc_control import CommandType -from osparc_control import PairedTransmitter - -command_add = CommandManifest( - action="add_internal_time", - description="adds internal time to a provided paramter", - params=[ - CommandParameter(name="a", description="param to add to internal time"), - ], - command_type=CommandType.WITH_DELAYED_REPLY, -) - -command_get_time = CommandManifest( - action="get_time", - description="gets the time", - params=[], - command_type=CommandType.WITH_IMMEDIATE_REPLY, -) - -command_print_solver_status = CommandManifest( - action="print_status", - description="prints the status of the solver", - params=[], - command_type=CommandType.WITHOUT_REPLY, -) - - -paired_transmitter = PairedTransmitter( - remote_host="localhost", - exposed_commands=[command_add, command_get_time, command_print_solver_status], - remote_port=1235, - listen_port=1234, -) diff --git a/examples/base_time_add/time_solver.py b/examples/base_time_add/time_solver.py deleted file mode 100644 index 0cb780d..0000000 --- a/examples/base_time_add/time_solver.py +++ /dev/null @@ -1,75 +0,0 @@ -from time import sleep - -from sidecar_solver import command_add -from sidecar_solver import command_get_time -from sidecar_solver import command_print_solver_status -from sidecar_solver import paired_transmitter - -from osparc_control.core import PairedTransmitter -from osparc_control.models import CommandRequest - - -def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: - if request.action == command_add.action: - sum_result = time_solver._add(**request.params) - time_solver.paired_transmitter.reply_to_command( - request_id=request.request_id, payload=sum_result - ) - return - - if request.action == command_get_time.action: - time_solver.paired_transmitter.reply_to_command( - request_id=request.request_id, payload=time_solver.time - ) - return - - if request.action == command_print_solver_status.action: - print("Solver internal status", time_solver) - # finally exit - time_solver.can_continue = False - return - - -class TimeSolver: - def __init__( - self, initial_time: float, paired_transmitter: PairedTransmitter - ) -> None: - self.time = initial_time - self.paired_transmitter = paired_transmitter - - # internal time tick - self.sleep_internal: float = 0.1 - self.can_continue: bool = True - - def __repr__(self) -> str: - return ( - f"<{self.__class__.__name__} time={self.time}, " - f"sleep_interval={self.sleep_internal}>" - ) - - def _add(self, a: float) -> float: - return self.time + a - - def run(self) -> None: - """main loop of the solver""" - while self.can_continue: - # process incoming requests from remote - for request in self.paired_transmitter.get_incoming_requests(): - handle_inputs(time_solver=self, request=request) - - # process internal stuff - self.time += 1 - sleep(self.sleep_internal) - - -def main() -> None: - paired_transmitter.start_background_sync() - - solver = TimeSolver(initial_time=0, paired_transmitter=paired_transmitter) - solver.run() - - paired_transmitter.stop_background_sync() - - -if __name__ == "__main__": - main() diff --git a/examples/simple/replyer.py b/examples/simple/replyer.py deleted file mode 100644 index fe97b0d..0000000 --- a/examples/simple/replyer.py +++ /dev/null @@ -1,44 +0,0 @@ -import random -import time - -from osparc_control import CommandManifest -from osparc_control import CommandParameter -from osparc_control import CommandType -from osparc_control import PairedTransmitter - - -# declare some commands to which a reply can be provided -random_in_range_manifest = CommandManifest( - action="random_in_range", - description="gets the time", - params=[ - CommandParameter(name="a", description="lower bound for random numbers"), - CommandParameter(name="b", description="upper bound for random numbers"), - ], - command_type=CommandType.WITH_IMMEDIATE_REPLY, -) - -paired_transmitter = PairedTransmitter( - remote_host="localhost", - exposed_commands=[random_in_range_manifest], - remote_port=2346, - listen_port=2345, -) -paired_transmitter.start_background_sync() - -wait_for_requests = True -while wait_for_requests: - for command in paired_transmitter.get_incoming_requests(): - if command.action == random_in_range_manifest.action: - random_int = random.randint( # noqa: S311 - command.params["a"], command.params["b"] - ) - paired_transmitter.reply_to_command( - request_id=command.request_id, payload=random_int - ) - wait_for_requests = False - -# allow for message to be delivered -time.sleep(0.01) - -paired_transmitter.stop_background_sync() diff --git a/examples/simple/requester.py b/examples/simple/requester.py deleted file mode 100644 index bf542f4..0000000 --- a/examples/simple/requester.py +++ /dev/null @@ -1,15 +0,0 @@ -from osparc_control import PairedTransmitter - -paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 -) - -# using a context manager allows to avoid calling -# paired_transmitter.start_background_sync() before making/receiving requests -# paired_transmitter.stop_background_sync() to close and cleanup when done -with paired_transmitter: - random_int = paired_transmitter.request_with_immediate_reply( - "random_in_range", timeout=10.0, params={"a": 1, "b": 10} - ) - print(random_int) - assert 1 <= random_int <= 10 # noqa: S101 diff --git a/src/osparc_control/__init__.py b/src/osparc_control/__init__.py index 869502f..0b747e1 100644 --- a/src/osparc_control/__init__.py +++ b/src/osparc_control/__init__.py @@ -2,6 +2,13 @@ from .core import PairedTransmitter from .models import CommandManifest from .models import CommandParameter +from .models import CommandRequest from .models import CommandType -__all__ = ["PairedTransmitter", "CommandManifest", "CommandParameter", "CommandType"] +__all__ = [ + "PairedTransmitter", + "CommandManifest", + "CommandParameter", + "CommandRequest", + "CommandType", +] From c147041c3dd39a4fd0bdd8d5afd31e94e7aba9e3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 15:53:37 +0200 Subject: [PATCH 69/86] added missing examples --- examples/1_simple/replyer.py | 44 +++++++++ examples/1_simple/requester.py | 15 ++++ examples/2_base_time_add/controller.py | 52 +++++++++++ examples/2_base_time_add/solver.py | 118 +++++++++++++++++++++++++ 4 files changed, 229 insertions(+) create mode 100644 examples/1_simple/replyer.py create mode 100644 examples/1_simple/requester.py create mode 100644 examples/2_base_time_add/controller.py create mode 100644 examples/2_base_time_add/solver.py diff --git a/examples/1_simple/replyer.py b/examples/1_simple/replyer.py new file mode 100644 index 0000000..fe97b0d --- /dev/null +++ b/examples/1_simple/replyer.py @@ -0,0 +1,44 @@ +import random +import time + +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommandType +from osparc_control import PairedTransmitter + + +# declare some commands to which a reply can be provided +random_in_range_manifest = CommandManifest( + action="random_in_range", + description="gets the time", + params=[ + CommandParameter(name="a", description="lower bound for random numbers"), + CommandParameter(name="b", description="upper bound for random numbers"), + ], + command_type=CommandType.WITH_IMMEDIATE_REPLY, +) + +paired_transmitter = PairedTransmitter( + remote_host="localhost", + exposed_commands=[random_in_range_manifest], + remote_port=2346, + listen_port=2345, +) +paired_transmitter.start_background_sync() + +wait_for_requests = True +while wait_for_requests: + for command in paired_transmitter.get_incoming_requests(): + if command.action == random_in_range_manifest.action: + random_int = random.randint( # noqa: S311 + command.params["a"], command.params["b"] + ) + paired_transmitter.reply_to_command( + request_id=command.request_id, payload=random_int + ) + wait_for_requests = False + +# allow for message to be delivered +time.sleep(0.01) + +paired_transmitter.stop_background_sync() diff --git a/examples/1_simple/requester.py b/examples/1_simple/requester.py new file mode 100644 index 0000000..bf542f4 --- /dev/null +++ b/examples/1_simple/requester.py @@ -0,0 +1,15 @@ +from osparc_control import PairedTransmitter + +paired_transmitter = PairedTransmitter( + remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 +) + +# using a context manager allows to avoid calling +# paired_transmitter.start_background_sync() before making/receiving requests +# paired_transmitter.stop_background_sync() to close and cleanup when done +with paired_transmitter: + random_int = paired_transmitter.request_with_immediate_reply( + "random_in_range", timeout=10.0, params={"a": 1, "b": 10} + ) + print(random_int) + assert 1 <= random_int <= 10 # noqa: S101 diff --git a/examples/2_base_time_add/controller.py b/examples/2_base_time_add/controller.py new file mode 100644 index 0000000..0010c1c --- /dev/null +++ b/examples/2_base_time_add/controller.py @@ -0,0 +1,52 @@ +from osparc_control import PairedTransmitter + + +def main() -> None: + paired_transmitter = PairedTransmitter( + remote_host="localhost", exposed_commands=[], remote_port=1234, listen_port=1235 + ) + + # using a context manager allows to avoid calling + # paired_transmitter.start_background_sync() before making/receiving requests + # paired_transmitter.stop_background_sync() to close and cleanup when done + with paired_transmitter: + + # call remote `add_internal_time`: + # - the user is required to `poll` for the result + # - returns: the internal time plus the provided parameter `a` + add_params = {"a": 10} + print(f"Will add internal time to parameter {add_params}") + request_id = paired_transmitter.request_with_delayed_reply( + "add_internal_time", params=add_params + ) + # very basic way of polling for expected result + has_result = False + result = None + while not has_result: + has_result, result = paired_transmitter.check_for_reply( + request_id=request_id + ) + print(f"result of `add_internal_time` with input {add_params}: {result}") + + # call remote `get_time`: + # - will block until the result is received + # - NOTE: timeout parameter is required and should be proportional + # to the mount of time the user expects the remote to reply in + # - returns: the server internal time + print("getting solver time") + solver_time = paired_transmitter.request_with_immediate_reply( + "get_time", timeout=1.0 + ) + print(f"solver time= {solver_time}") + + # call remote `close_solver`: + # - does not return anything + # - will cause the solver.py to close + print("sending command to close remote") + paired_transmitter.request_without_reply("close_solver") + + print("finished") + + +if __name__ == "__main__": + main() diff --git a/examples/2_base_time_add/solver.py b/examples/2_base_time_add/solver.py new file mode 100644 index 0000000..c564511 --- /dev/null +++ b/examples/2_base_time_add/solver.py @@ -0,0 +1,118 @@ +from time import sleep + +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommandRequest +from osparc_control import CommandType +from osparc_control import PairedTransmitter + +# Define exposed commands by this service. +# Each command can be of 3 separate types. + +# The requester is expected to poll for the results of this command. +# The results will be delivered as soon as possible and stored +# on the requester's side until he is ready to use it. +# NOTE: when handling this command a respone should always be provided. +COMMAND_ADD = CommandManifest( + action="add_internal_time", + description="adds internal time to a provided paramter", + params=[ + CommandParameter(name="a", description="param to add to internal time"), + ], + command_type=CommandType.WITH_DELAYED_REPLY, +) + +# The requester will be blocked until the result to this comand is delivered. +# NOTE: when handling this command a respone should always be provided. +# NOTE: the requester waits with a timeout which has to be proportional to +# how much time it takes for this command to be processed +COMMAND_GET_TIME = CommandManifest( + action="get_time", + description="gets the time", + params=[], + command_type=CommandType.WITH_IMMEDIATE_REPLY, +) + +# The requester expects no reply to this command. +# NOTE: when handling this command a reply must NOT be provided. +COMMAND_CLOSE_SOLVER = CommandManifest( + action="close_solver", + description="prints the status of the solver", + params=[], + command_type=CommandType.WITHOUT_REPLY, +) + + +def handle_inputs(time_solver: "TimeSolver", request: CommandRequest) -> None: + """ + Handle incoming requests agains declared commands + """ + if request.action == COMMAND_ADD.action: + sum_result = time_solver._add(**request.params) + time_solver.paired_transmitter.reply_to_command( + request_id=request.request_id, payload=sum_result + ) + return + + if request.action == COMMAND_GET_TIME.action: + time_solver.paired_transmitter.reply_to_command( + request_id=request.request_id, payload=time_solver.time + ) + return + + if request.action == COMMAND_CLOSE_SOLVER.action: + print(f"Quitting solver: {time_solver}") + # signal solver loop to stop + time_solver.can_continue = False + return + + +class TimeSolver: + def __init__( + self, initial_time: float, paired_transmitter: PairedTransmitter + ) -> None: + self.time = initial_time + self.paired_transmitter = paired_transmitter + + # internal time tick + self.sleep_internal: float = 0.1 + self.can_continue: bool = True + + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__} time={self.time}, " + f"sleep_interval={self.sleep_internal}>" + ) + + def _add(self, a: float) -> float: + return self.time + a + + def run(self) -> None: + """main loop of the solver""" + while self.can_continue: + # process incoming requests from remote + for request in self.paired_transmitter.get_incoming_requests(): + handle_inputs(time_solver=self, request=request) + + # solver internal processing + self.time += 1 + sleep(self.sleep_internal) + + +def main() -> None: + paired_transmitter = PairedTransmitter( + remote_host="localhost", + exposed_commands=[COMMAND_ADD, COMMAND_GET_TIME, COMMAND_CLOSE_SOLVER], + remote_port=1235, + listen_port=1234, + ) + + with paired_transmitter: + time_solver = TimeSolver(initial_time=0, paired_transmitter=paired_transmitter) + time_solver.run() + + print("finished") + + +if __name__ == "__main__": + main() From 24532070b2ad9d792200dc7459646906dd86d23f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 16:29:15 +0200 Subject: [PATCH 70/86] add examples tester --- examples/1_simple/replyer.py | 44 ------------------------------------ 1 file changed, 44 deletions(-) delete mode 100644 examples/1_simple/replyer.py diff --git a/examples/1_simple/replyer.py b/examples/1_simple/replyer.py deleted file mode 100644 index fe97b0d..0000000 --- a/examples/1_simple/replyer.py +++ /dev/null @@ -1,44 +0,0 @@ -import random -import time - -from osparc_control import CommandManifest -from osparc_control import CommandParameter -from osparc_control import CommandType -from osparc_control import PairedTransmitter - - -# declare some commands to which a reply can be provided -random_in_range_manifest = CommandManifest( - action="random_in_range", - description="gets the time", - params=[ - CommandParameter(name="a", description="lower bound for random numbers"), - CommandParameter(name="b", description="upper bound for random numbers"), - ], - command_type=CommandType.WITH_IMMEDIATE_REPLY, -) - -paired_transmitter = PairedTransmitter( - remote_host="localhost", - exposed_commands=[random_in_range_manifest], - remote_port=2346, - listen_port=2345, -) -paired_transmitter.start_background_sync() - -wait_for_requests = True -while wait_for_requests: - for command in paired_transmitter.get_incoming_requests(): - if command.action == random_in_range_manifest.action: - random_int = random.randint( # noqa: S311 - command.params["a"], command.params["b"] - ) - paired_transmitter.reply_to_command( - request_id=command.request_id, payload=random_int - ) - wait_for_requests = False - -# allow for message to be delivered -time.sleep(0.01) - -paired_transmitter.stop_background_sync() From 77bbe64e1751e28b30ab1a305d001bae4dfaef53 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 16:32:07 +0200 Subject: [PATCH 71/86] add examples tester --- examples/1_simple/replier.py | 44 +++++++++++++++++++++ tests/test_examples.py | 77 ++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 examples/1_simple/replier.py create mode 100644 tests/test_examples.py diff --git a/examples/1_simple/replier.py b/examples/1_simple/replier.py new file mode 100644 index 0000000..fe97b0d --- /dev/null +++ b/examples/1_simple/replier.py @@ -0,0 +1,44 @@ +import random +import time + +from osparc_control import CommandManifest +from osparc_control import CommandParameter +from osparc_control import CommandType +from osparc_control import PairedTransmitter + + +# declare some commands to which a reply can be provided +random_in_range_manifest = CommandManifest( + action="random_in_range", + description="gets the time", + params=[ + CommandParameter(name="a", description="lower bound for random numbers"), + CommandParameter(name="b", description="upper bound for random numbers"), + ], + command_type=CommandType.WITH_IMMEDIATE_REPLY, +) + +paired_transmitter = PairedTransmitter( + remote_host="localhost", + exposed_commands=[random_in_range_manifest], + remote_port=2346, + listen_port=2345, +) +paired_transmitter.start_background_sync() + +wait_for_requests = True +while wait_for_requests: + for command in paired_transmitter.get_incoming_requests(): + if command.action == random_in_range_manifest.action: + random_int = random.randint( # noqa: S311 + command.params["a"], command.params["b"] + ) + paired_transmitter.reply_to_command( + request_id=command.request_id, payload=random_int + ) + wait_for_requests = False + +# allow for message to be delivered +time.sleep(0.01) + +paired_transmitter.stop_background_sync() diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..cd63cb9 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,77 @@ +import itertools +import sys +from pathlib import Path +from subprocess import PIPE # noqa: S404 +from subprocess import Popen # noqa: S404 +from subprocess import STDOUT # noqa: S404 +from time import sleep +from typing import List + +import pytest + +HERE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +# FIXTURES + + +@pytest.fixture +def path_1_simple() -> Path: + path = (HERE / ".." / "examples" / "1_simple").resolve() + assert path.exists() + return path + + +@pytest.fixture +def path_2_base_time_add() -> Path: + path = (HERE / ".." / "examples" / "2_base_time_add").resolve() + assert path.exists() + return path + + +# UTILS + + +def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: + # check all exist before running + for script_to_run in scrips_to_run: + assert script_to_run.exists() + + for permutation in itertools.permutations(scrips_to_run): + # run provided scripts + processes = [ + Popen( # noqa: S607 + ["python", f"{x}"], shell=True, stdout=PIPE, stderr=STDOUT # noqa: S602 + ) + for x in permutation + ] + + # wait for processes to finish + continue_running = True + while continue_running: + continue_running = all( + [process.returncode is not None for process in processes] + ) + sleep(0.3) + + # ensure all processes finished successfully + for process in processes: + stdout, _ = process.communicate() + assert process.returncode == 0, stdout.decode() + + +# TESTS + + +def test_example_1_simple_runs(path_1_simple: Path) -> None: + replier_path = path_1_simple / "replier.py" + requester_path = path_1_simple / "requester.py" + + assert_run_in_parallel([replier_path, requester_path]) + + +def test_example_2_base_time_add_runs(path_2_base_time_add: Path) -> None: + controller_path = path_2_base_time_add / "controller.py" + solver_path = path_2_base_time_add / "solver.py" + + assert_run_in_parallel([controller_path, solver_path]) From 62a26c6303ba4285d4742ae3df0e11a0b2cc200d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 16:41:46 +0200 Subject: [PATCH 72/86] add comment to test --- tests/test_examples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_examples.py b/tests/test_examples.py index cd63cb9..597fcc0 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -39,6 +39,7 @@ def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: for permutation in itertools.permutations(scrips_to_run): # run provided scripts + print(f"Running permutation {permutation}") processes = [ Popen( # noqa: S607 ["python", f"{x}"], shell=True, stdout=PIPE, stderr=STDOUT # noqa: S602 From c824be8e001ded0f3ed2407bf897a4231ce55af2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 16:44:58 +0200 Subject: [PATCH 73/86] give more time to deliver messages --- examples/1_simple/replier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/1_simple/replier.py b/examples/1_simple/replier.py index fe97b0d..604023c 100644 --- a/examples/1_simple/replier.py +++ b/examples/1_simple/replier.py @@ -39,6 +39,6 @@ wait_for_requests = False # allow for message to be delivered -time.sleep(0.01) +time.sleep(0.3) paired_transmitter.stop_background_sync() From 3e5d80ee465ef34c5c08a50e40d5fdef1d12fd6d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Apr 2022 16:46:24 +0200 Subject: [PATCH 74/86] added extra test --- examples/1_simple/requester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/1_simple/requester.py b/examples/1_simple/requester.py index bf542f4..8f603fd 100644 --- a/examples/1_simple/requester.py +++ b/examples/1_simple/requester.py @@ -12,4 +12,5 @@ "random_in_range", timeout=10.0, params={"a": 1, "b": 10} ) print(random_int) + assert random_int is not None # noqa: S101 assert 1 <= random_int <= 10 # noqa: S101 From 1735dd6c260aa63be55b783eaffce0fdf77228b2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 10:34:52 +0200 Subject: [PATCH 75/86] replaced sphinx with docsify --- .github/workflows/tests.yml | 8 - .gitignore | 1 + .nojekyll | 0 .readthedocs.yml | 12 -- CODE_OF_CONDUCT.rst => CODE_OF_CONDUCT.md | 51 ++----- CONTRIBUTING.rst => CONTRIBUTING.md | 96 +++++------- LICENSE.rst => LICENSE.md | 5 +- README.md | 102 +++++++++++++ README.rst | 96 ------------ docs/codeofconduct.rst | 1 - docs/conf.py | 13 -- docs/contributing.rst | 4 - docs/index.rst | 16 -- docs/license.rst | 1 - docs/reference.rst | 22 --- docs/requirements.txt | 3 - docs/usage.rst | 20 --- examples/1_simple/requester.py | 6 +- index.html | 28 ++++ noxfile.py | 40 +---- poetry.lock | 173 +++++----------------- pyproject.toml | 5 +- 22 files changed, 233 insertions(+), 470 deletions(-) create mode 100644 .nojekyll delete mode 100644 .readthedocs.yml rename CODE_OF_CONDUCT.rst => CODE_OF_CONDUCT.md (87%) rename CONTRIBUTING.rst => CONTRIBUTING.md (55%) rename LICENSE.rst => LICENSE.md (94%) create mode 100644 README.md delete mode 100644 README.rst delete mode 100644 docs/codeofconduct.rst delete mode 100644 docs/conf.py delete mode 100644 docs/contributing.rst delete mode 100644 docs/index.rst delete mode 100644 docs/license.rst delete mode 100644 docs/reference.rst delete mode 100644 docs/requirements.txt delete mode 100644 docs/usage.rst create mode 100644 index.html diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0da19ee..8a69c2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,6 @@ jobs: - { python: "3.10", os: "macos-latest", session: "tests" } - { python: "3.10", os: "ubuntu-latest", session: "typeguard" } - { python: "3.10", os: "ubuntu-latest", session: "xdoctest" } - - { python: "3.6", os: "ubuntu-latest", session: "docs-build" } env: NOXSESSION: ${{ matrix.session }} @@ -104,13 +103,6 @@ jobs: name: coverage-data path: ".coverage.*" - - name: Upload documentation - if: matrix.session == 'docs-build' - uses: actions/upload-artifact@v2.2.4 - with: - name: docs - path: docs/_build - coverage: runs-on: ubuntu-latest needs: tests diff --git a/.gitignore b/.gitignore index 337f9ef..3013fd7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /src/*.egg-info/ __pycache__/ *ignore* +.vscode/ diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 66f2a21..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -build: - os: ubuntu-20.04 - tools: - python: "3.10" -sphinx: - configuration: docs/conf.py -formats: all -python: - install: - - requirements: docs/requirements.txt - - path: . diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.md similarity index 87% rename from CODE_OF_CONDUCT.rst rename to CODE_OF_CONDUCT.md index 797592e..2b1f166 100644 --- a/CODE_OF_CONDUCT.rst +++ b/CODE_OF_CONDUCT.md @@ -1,16 +1,12 @@ -Contributor Covenant Code of Conduct -==================================== +# Contributor Covenant Code of Conduct -Our Pledge ----------- +## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. - -Our Standards -------------- +## Our Standards Examples of behavior that contributes to a positive environment for our community include: @@ -31,75 +27,56 @@ Examples of unacceptable behavior include: - Other conduct which could reasonably be considered inappropriate in a professional setting -Enforcement Responsibilities ----------------------------- +## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. - -Scope ------ +## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. - -Enforcement ------------ +## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at neagu@itis.swiss. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. - -Enforcement Guidelines ----------------------- +## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: - -1. Correction -~~~~~~~~~~~~~ +### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. - -2. Warning -~~~~~~~~~~ +### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. - -3. Temporary Ban -~~~~~~~~~~~~~~~~ +### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. - -4. Permanent Ban -~~~~~~~~~~~~~~~~ +### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. +## Attribution -Attribution ------------ - -This Code of Conduct is adapted from the `Contributor Covenant `__, version 2.0, +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. -Community Impact Guidelines were inspired by `Mozilla’s code of conduct enforcement ladder `__. - -.. _homepage: https://www.contributor-covenant.org +Community Impact Guidelines were inspired by [Mozilla’s code of conduct enforcement ladder](https://github.com/mozilla/diversity). For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.md similarity index 55% rename from CONTRIBUTING.rst rename to CONTRIBUTING.md index 9580078..6dc16c1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.md @@ -1,26 +1,19 @@ -Contributor Guide -================= +# Contributor Guide Thank you for your interest in improving this project. -This project is open-source under the `MIT license`_ and +This project is open-source under the [MIT license] and welcomes contributions in the form of bug reports, feature requests, and pull requests. Here is a list of important resources for contributors: -- `Source Code`_ -- `Documentation`_ -- `Issue Tracker`_ -- `Code of Conduct`_ +- [Source Code] +- [Documentation] +- [Issue Tracker] +- [Code of Conduct] -.. _MIT license: https://opensource.org/licenses/MIT -.. _Source Code: https://github.com/ITISFoundation/osparc-control -.. _Documentation: https://osparc-control.readthedocs.io/ -.. _Issue Tracker: https://github.com/ITISFoundation/osparc-control/issues +## How to report a bug -How to report a bug -------------------- - -Report bugs on the `Issue Tracker`_. +Report bugs on the [Issue Tracker]. When filing an issue, make sure to answer these questions: @@ -33,73 +26,59 @@ When filing an issue, make sure to answer these questions: The best way to get your bug fixed is to provide a test case, and/or steps to reproduce the issue. +## How to request a feature -How to request a feature ------------------------- - -Request features on the `Issue Tracker`_. +Request features on the [Issue Tracker]. - -How to set up your development environment ------------------------------------------- +## How to set up your development environment You need Python 3.6+ and the following tools: -- Poetry_ -- Nox_ -- nox-poetry_ +- [Poetry] +- [Nox] +- [nox-poetry] Install the package with development requirements: -.. code:: console - +```bash $ poetry install +``` You can now run an interactive Python session, or the command-line interface: -.. code:: console - +```bash $ poetry run python $ poetry run osparc-control +``` -.. _Poetry: https://python-poetry.org/ -.. _Nox: https://nox.thea.codes/ -.. _nox-poetry: https://nox-poetry.readthedocs.io/ - - -How to test the project ------------------------ +## How to test the project Run the full test suite: -.. code:: console - +```bash $ nox +``` List the available Nox sessions: -.. code:: console - +```bash $ nox --list-sessions +``` You can also run a specific Nox session. For example, invoke the unit test suite like this: -.. code:: console - +```bash $ nox --session=tests +``` -Unit tests are located in the ``tests`` directory, -and are written using the pytest_ testing framework. +Unit tests are located in the `tests` directory, +and are written using the [pytest] testing framework. -.. _pytest: https://pytest.readthedocs.io/ +## How to submit changes - -How to submit changes ---------------------- - -Open a `pull request`_ to submit changes to this project. +Open a [pull request] to submit changes to this project. Your pull request needs to meet the following guidelines for acceptance: @@ -111,13 +90,20 @@ Feel free to submit early, though—we can always iterate on this. To run linting and code formatting checks before committing your change, you can install pre-commit as a Git hook by running the following command: -.. code:: console - +```bash $ nox --session=pre-commit -- install +``` It is recommended to open an issue before starting work on anything. This will allow a chance to talk it over with the owners and validate your approach. -.. _pull request: https://github.com/ITISFoundation/osparc-control/pulls -.. github-only -.. _Code of Conduct: CODE_OF_CONDUCT.rst +[mit license]: https://opensource.org/licenses/MIT +[source code]: https://github.com/ITISFoundation/osparc-control +[documentation]: https://osparc-control.readthedocs.io/ +[issue tracker]: https://github.com/ITISFoundation/osparc-control/issues +[code of conduct]: CODE_OF_CONDUCT.rst +[pull request]: https://github.com/ITISFoundation/osparc-control/pulls +[poetry]: https://python-poetry.org/ +[nox]: https://nox.thea.codes/ +[nox-poetry]: https://nox-poetry.readthedocs.io/ +[pytest]: https://pytest.readthedocs.io/ diff --git a/LICENSE.rst b/LICENSE.md similarity index 94% rename from LICENSE.rst rename to LICENSE.md index 656cf21..852d08c 100644 --- a/LICENSE.rst +++ b/LICENSE.md @@ -1,7 +1,6 @@ -MIT License -=========== +# MIT License -Copyright © 2022 Andrei Neagu +Copyright © 2022 ITIS Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md new file mode 100644 index 0000000..2276b16 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# Osparc Control + +[![PyPI](https://img.shields.io/pypi/v/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Status](https://img.shields.io/pypi/status/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Python Version](https://img.shields.io/pypi/pyversions/osparc-control)](https://pypi.org/project/osparc-control) [![License](https://img.shields.io/pypi/l/osparc-control)](https://opensource.org/licenses/MIT) + +[![Read the documentation at https://osparc-control.readthedocs.io/](https://img.shields.io/readthedocs/osparc-control/latest.svg?label=Read%20the%20Docs)](https://osparc-control.readthedocs.io/) [![Tests](https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg)](https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests) + +[![Codecov](https://codecov.io/gh/ITISFoundation/osparc-control/branch/main/graph/badge.svg)](https://codecov.io/gh/ITISFoundation/osparc-control) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +--- + +## Installation + +You can install _Osparc Control_ via [pip] from [PyPI]: + +```bash +pip install osparc-control +``` + +## Examples + +To run below examples either clone the repo or copy code from the snippets +below the commands + +### Simple example + +A first example where `requester.py` asks for a random number and +`replier.py` defines an interface to provide it. + +- In a first terminal run: + +```bash +python examples/1_simple/requester.py +``` + +#### examples/1_simple/requester.py + +[filename](examples/1_simple/requester.py ":include :type=code") + +- In a second terminal run: + +```bash +python examples/1_simple/replier.py +``` + +#### examples/1_simple/replier.py + +[filename](examples/1_simple/replier.py ":include :type=code") + +### Advanced example + +A showcase of all the types of supported requests. + +- In a first terminal run: + +```bash +python examples/2_base_time_add/controller.py +``` + +#### examples/2_base_time_add/controller.py + +[filename](examples/2_base_time_add/controller.py ":include :type=code") + +- In a second terminal run: + +```bash +python examples/2_base_time_add/solver.py +``` + +#### examples/2_base_time_add/solver.py + +[filename](examples/2_base_time_add/solver.py ":include :type=code") + +## Contributing + +Contributions are very welcome. +To learn more, see the [Contributor Guide]. +Our [Code of Conduct] pledge. + +## License + +Distributed under the terms of the [MIT license], +_Osparc Control_ is free and open source software. + +## Issues + +If you encounter any problems, +please [file an issue] along with a detailed description. + +## Credits + +This project was generated from [@cjolowicz]'s [Hypermodern Python Cookiecutter] template. + +[@cjolowicz]: https://github.com/cjolowicz +[cookiecutter]: https://github.com/audreyr/cookiecutter +[mit license]: LICENSE.md +[pypi]: https://pypi.org/ +[hypermodern python cookiecutter]: https://github.com/cjolowicz/cookiecutter-hypermodern-python +[file an issue]: https://github.com/ITISFoundation/osparc-control/issues +[pip]: https://pip.pypa.io/ +[contributor guide]: CONTRIBUTING.md +[code of conduct]: CODE_OF_CONDUCT.md +[usage]: https://osparc-control.readthedocs.io/en/latest/usage.html diff --git a/README.rst b/README.rst deleted file mode 100644 index 0f74e28..0000000 --- a/README.rst +++ /dev/null @@ -1,96 +0,0 @@ -Osparc Control -============== - -|PyPI| |Status| |Python Version| |License| - -|Read the Docs| |Tests| |Codecov| - -|pre-commit| |Black| - -.. |PyPI| image:: https://img.shields.io/pypi/v/osparc-control.svg - :target: https://pypi.org/project/osparc-control/ - :alt: PyPI -.. |Status| image:: https://img.shields.io/pypi/status/osparc-control.svg - :target: https://pypi.org/project/osparc-control/ - :alt: Status -.. |Python Version| image:: https://img.shields.io/pypi/pyversions/osparc-control - :target: https://pypi.org/project/osparc-control - :alt: Python Version -.. |License| image:: https://img.shields.io/pypi/l/osparc-control - :target: https://opensource.org/licenses/MIT - :alt: License -.. |Read the Docs| image:: https://img.shields.io/readthedocs/osparc-control/latest.svg?label=Read%20the%20Docs - :target: https://osparc-control.readthedocs.io/ - :alt: Read the documentation at https://osparc-control.readthedocs.io/ -.. |Tests| image:: https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg - :target: https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests - :alt: Tests -.. |Codecov| image:: https://codecov.io/gh/ITISFoundation/osparc-control/branch/main/graph/badge.svg - :target: https://codecov.io/gh/ITISFoundation/osparc-control - :alt: Codecov -.. |pre-commit| image:: https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white - :target: https://github.com/pre-commit/pre-commit - :alt: pre-commit -.. |Black| image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Black - - -Features --------- - -* TODO - - -Installation ------------- - -You can install *Osparc Control* via pip_ from PyPI_: - -.. code:: console - - $ pip install osparc-control - - -Usage ------ - -Please have a look at the examples folder. - - -Contributing ------------- - -Contributions are very welcome. -To learn more, see the `Contributor Guide`_. - - -License -------- - -Distributed under the terms of the `MIT license`_, -*Osparc Control* is free and open source software. - - -Issues ------- - -If you encounter any problems, -please `file an issue`_ along with a detailed description. - - -Credits -------- - -This project was generated from `@cjolowicz`_'s `Hypermodern Python Cookiecutter`_ template. - -.. _@cjolowicz: https://github.com/cjolowicz -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _MIT license: https://opensource.org/licenses/MIT -.. _PyPI: https://pypi.org/ -.. _Hypermodern Python Cookiecutter: https://github.com/cjolowicz/cookiecutter-hypermodern-python -.. _file an issue: https://github.com/ITISFoundation/osparc-control/issues -.. _pip: https://pip.pypa.io/ -.. github-only -.. _Contributor Guide: CONTRIBUTING.rst -.. _Usage: https://osparc-control.readthedocs.io/en/latest/usage.html diff --git a/docs/codeofconduct.rst b/docs/codeofconduct.rst deleted file mode 100644 index 96e0ba2..0000000 --- a/docs/codeofconduct.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CODE_OF_CONDUCT.rst diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 4770023..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Sphinx configuration.""" -from datetime import datetime - -project = "Osparc Control" -author = "Andrei Neagu" -copyright = f"{datetime.now().year}, {author}" -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx_click", -] -autodoc_typehints = "description" -html_theme = "furo" diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index c8670b6..0000000 --- a/docs/contributing.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. include:: ../CONTRIBUTING.rst - :end-before: github-only - -.. _Code of Conduct: codeofconduct.html diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 9d1fe91..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. include:: ../README.rst - :end-before: github-only - -.. _Contributor Guide: contributing.html -.. _Usage: usage.html - -.. toctree:: - :hidden: - :maxdepth: 1 - - usage - reference - contributing - Code of Conduct - License - Changelog diff --git a/docs/license.rst b/docs/license.rst deleted file mode 100644 index 68c5792..0000000 --- a/docs/license.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../LICENSE.rst diff --git a/docs/reference.rst b/docs/reference.rst deleted file mode 100644 index f75998d..0000000 --- a/docs/reference.rst +++ /dev/null @@ -1,22 +0,0 @@ -Reference -========= - - -osparc_control --------------- - -.. autoclass:: osparc_control.PairedTransmitter - :members: - :undoc-members: - -.. autoclass:: osparc_control.CommandManifest - :members: - :undoc-members: - -.. autoclass:: osparc_control.CommandParameter - :members: - :undoc-members: - -.. autoclass:: osparc_control.CommandType - :members: - :undoc-members: diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 4b4a2ff..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -furo==2021.11.23 -sphinx==4.3.0 -sphinx-click==3.0.2 diff --git a/docs/usage.rst b/docs/usage.rst deleted file mode 100644 index 322a3f2..0000000 --- a/docs/usage.rst +++ /dev/null @@ -1,20 +0,0 @@ -Usage -===== - -Please have a look at the examples folder. - - -Minimual examples -================= - -In one terminal run ``python examples/1_simple/requester.py``: - -.. literalinclude:: ../examples/1_simple/requester.py - :language: python - - - -In a second terminal run ``python examples/1_simple/replyer.py``: - -.. literalinclude:: ../examples/1_simple/replyer.py - :language: python diff --git a/examples/1_simple/requester.py b/examples/1_simple/requester.py index 8f603fd..06b3a17 100644 --- a/examples/1_simple/requester.py +++ b/examples/1_simple/requester.py @@ -1,7 +1,11 @@ from osparc_control import PairedTransmitter paired_transmitter = PairedTransmitter( - remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346 + remote_host="localhost", + exposed_commands=[], # not required to define an interface + # since both `PairedTransmitter`s run on the same host ports need to different + remote_port=2345, + listen_port=2346, ) # using a context manager allows to avoid calling diff --git a/index.html b/index.html new file mode 100644 index 0000000..093e592 --- /dev/null +++ b/index.html @@ -0,0 +1,28 @@ + + + + + + + + + + +
+ + + + + + diff --git a/noxfile.py b/noxfile.py index cc91fe2..4b12762 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,7 +1,7 @@ """Nox sessions.""" import os -import shutil import sys +from http import server from pathlib import Path from textwrap import dedent @@ -29,7 +29,6 @@ "tests", "typeguard", "xdoctest", - "docs-build", ) @@ -117,7 +116,7 @@ def safety(session: Session) -> None: @session(python=python_versions) # type: ignore def mypy(session: Session) -> None: """Type-check using mypy.""" - args = session.posargs or ["src", "tests", "docs/conf.py"] + args = session.posargs or ["src", "tests"] session.install(".") session.install("mypy", "pytest") session.run("mypy", *args) @@ -173,38 +172,7 @@ def xdoctest(session: Session) -> None: session.run("python", "-m", "xdoctest", *args) -@session(name="docs-build", python="3.6") # type: ignore -def docs_build(session: Session) -> None: - """Build the documentation.""" - args = session.posargs or ["docs", "docs/_build"] - if not session.posargs and "FORCE_COLOR" in os.environ: - args.insert(0, "--color") - - session.install(".") - session.install("sphinx", "sphinx-click", "furo") - - build_dir = Path("docs", "_build") - if build_dir.exists(): - shutil.rmtree(build_dir) - - session.run("sphinx-build", *args) - - @session(python="3.6") # type: ignore def docs(session: Session) -> None: - """Build and serve the documentation with live reloading on file changes.""" - args = session.posargs or [ - "--host", - "0.0.0.0", # noqa: S104 - "--open-browser", - "docs", - "docs/_build", - ] - session.install(".") - session.install("sphinx", "sphinx-autobuild", "sphinx-click", "furo") - - build_dir = Path("docs", "_build") - if build_dir.exists(): - shutil.rmtree(build_dir) - - session.run("sphinx-autobuild", *args) + """serve documentation locally""" + server.test(server.SimpleHTTPRequestHandler, port=3000) # type: ignore diff --git a/poetry.lock b/poetry.lock index d03b114..1df13b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -81,7 +81,7 @@ lxml = ["lxml"] [[package]] name = "black" -version = "22.1.0" +version = "22.3.0" description = "The uncompromising code formatter." category = "dev" optional = false @@ -93,7 +93,7 @@ dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} @@ -438,18 +438,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "livereload" -version = "2.6.3" -description = "Python LiveReload is an awesome tool for web developers" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -six = "*" -tornado = {version = "*", markers = "python_version > \"2.7\""} - [[package]] name = "markupsafe" version = "2.0.1" @@ -701,7 +689,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm [[package]] name = "pytz" -version = "2021.3" +version = "2022.1" description = "World timezone definitions, modern and historical" category = "dev" optional = false @@ -852,7 +840,7 @@ python-versions = "*" [[package]] name = "soupsieve" -version = "2.3.1" +version = "2.3.2" description = "A modern CSS selector implementation for Beautiful Soup." category = "dev" optional = false @@ -889,35 +877,6 @@ docs = ["sphinxcontrib-websupport"] lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.920)", "docutils-stubs", "types-typed-ast", "types-pkg-resources", "types-requests"] test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] -[[package]] -name = "sphinx-autobuild" -version = "2021.3.14" -description = "Rebuild Sphinx documentation on changes, with live-reload in the browser." -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -colorama = "*" -livereload = "*" -sphinx = "*" - -[package.extras] -test = ["pytest", "pytest-cov"] - -[[package]] -name = "sphinx-click" -version = "3.1.0" -description = "Sphinx extension that automatically documents click applications" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -click = ">=7.0" -docutils = "*" -sphinx = ">=2.0" - [[package]] name = "sphinxcontrib-applehelp" version = "1.0.2" @@ -1036,14 +995,6 @@ category = "dev" optional = false python-versions = ">=3.6" -[[package]] -name = "tornado" -version = "6.1" -description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "dev" -optional = false -python-versions = ">= 3.5" - [[package]] name = "typed-ast" version = "1.4.3" @@ -1095,7 +1046,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.13.3" +version = "20.14.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1148,7 +1099,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "8c9ff7190cd2d985bff1a6636e8f54011318ae3afee70ede8e37a8a3504b1e78" +content-hash = "f1f6ce94e1b130e96d40a35dd42ad9f7af8a51e49aa6af0c30c4351324176ad5" [metadata.files] alabaster = [ @@ -1180,29 +1131,29 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, ] black = [ - {file = "black-22.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1297c63b9e1b96a3d0da2d85d11cd9bf8664251fd69ddac068b98dc4f34f73b6"}, - {file = "black-22.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2ff96450d3ad9ea499fc4c60e425a1439c2120cbbc1ab959ff20f7c76ec7e866"}, - {file = "black-22.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e21e1f1efa65a50e3960edd068b6ae6d64ad6235bd8bfea116a03b21836af71"}, - {file = "black-22.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2f69158a7d120fd641d1fa9a921d898e20d52e44a74a6fbbcc570a62a6bc8ab"}, - {file = "black-22.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:228b5ae2c8e3d6227e4bde5920d2fc66cc3400fde7bcc74f480cb07ef0b570d5"}, - {file = "black-22.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1a5ed73ab4c482208d20434f700d514f66ffe2840f63a6252ecc43a9bc77e8a"}, - {file = "black-22.1.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35944b7100af4a985abfcaa860b06af15590deb1f392f06c8683b4381e8eeaf0"}, - {file = "black-22.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7835fee5238fc0a0baf6c9268fb816b5f5cd9b8793423a75e8cd663c48d073ba"}, - {file = "black-22.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dae63f2dbf82882fa3b2a3c49c32bffe144970a573cd68d247af6560fc493ae1"}, - {file = "black-22.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa1db02410b1924b6749c245ab38d30621564e658297484952f3d8a39fce7e8"}, - {file = "black-22.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c8226f50b8c34a14608b848dc23a46e5d08397d009446353dad45e04af0c8e28"}, - {file = "black-22.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2d6f331c02f0f40aa51a22e479c8209d37fcd520c77721c034517d44eecf5912"}, - {file = "black-22.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:742ce9af3086e5bd07e58c8feb09dbb2b047b7f566eb5f5bc63fd455814979f3"}, - {file = "black-22.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fdb8754b453fb15fad3f72cd9cad3e16776f0964d67cf30ebcbf10327a3777a3"}, - {file = "black-22.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5660feab44c2e3cb24b2419b998846cbb01c23c7fe645fee45087efa3da2d61"}, - {file = "black-22.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:6f2f01381f91c1efb1451998bd65a129b3ed6f64f79663a55fe0e9b74a5f81fd"}, - {file = "black-22.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:efbadd9b52c060a8fc3b9658744091cb33c31f830b3f074422ed27bad2b18e8f"}, - {file = "black-22.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8871fcb4b447206904932b54b567923e5be802b9b19b744fdff092bd2f3118d0"}, - {file = "black-22.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ccad888050f5393f0d6029deea2a33e5ae371fd182a697313bdbd835d3edaf9c"}, - {file = "black-22.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07e5c049442d7ca1a2fc273c79d1aecbbf1bc858f62e8184abe1ad175c4f7cc2"}, - {file = "black-22.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:373922fc66676133ddc3e754e4509196a8c392fec3f5ca4486673e685a421321"}, - {file = "black-22.1.0-py3-none-any.whl", hash = "sha256:3524739d76b6b3ed1132422bf9d82123cd1705086723bc3e235ca39fd21c667d"}, - {file = "black-22.1.0.tar.gz", hash = "sha256:a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] cached-property = [ {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, @@ -1416,9 +1367,6 @@ jinja2 = [ {file = "Jinja2-3.0.3-py3-none-any.whl", hash = "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8"}, {file = "Jinja2-3.0.3.tar.gz", hash = "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"}, ] -livereload = [ - {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, -] markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, @@ -1629,8 +1577,8 @@ pytest = [ {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] pytz = [ - {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"}, - {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"}, + {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, + {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, ] pyupgrade = [ {file = "pyupgrade-2.31.0-py2.py3-none-any.whl", hash = "sha256:0a62c5055f854d7f36e155b7ee8920561bf0399c53edd975cf02436eef8937fc"}, @@ -1783,21 +1731,13 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] soupsieve = [ - {file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"}, - {file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"}, + {file = "soupsieve-2.3.2-py3-none-any.whl", hash = "sha256:a714129d3021ec17ce5be346b1007300558b378332c289a1a20e7d4de6ff18a5"}, + {file = "soupsieve-2.3.2.tar.gz", hash = "sha256:0bcc6d7432153063e3df09c3ac9442af3eba488715bfcad6a4c38ccb2a523124"}, ] sphinx = [ {file = "Sphinx-4.3.2-py3-none-any.whl", hash = "sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851"}, {file = "Sphinx-4.3.2.tar.gz", hash = "sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c"}, ] -sphinx-autobuild = [ - {file = "sphinx-autobuild-2021.3.14.tar.gz", hash = "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05"}, - {file = "sphinx_autobuild-2021.3.14-py3-none-any.whl", hash = "sha256:8fe8cbfdb75db04475232f05187c776f46f6e9e04cacf1e49ce81bdac649ccac"}, -] -sphinx-click = [ - {file = "sphinx-click-3.1.0.tar.gz", hash = "sha256:36dbf271b1d2600fb05bd598ddeed0b6b6acf35beaf8bc9d507ba7716b232b0e"}, - {file = "sphinx_click-3.1.0-py3-none-any.whl", hash = "sha256:8fb0b048a577d346d741782e44d041d7e908922858273d99746f305870116121"}, -] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, @@ -1842,49 +1782,6 @@ tomli = [ {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] -tornado = [ - {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675"}, - {file = "tornado-6.1-cp35-cp35m-win32.whl", hash = "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5"}, - {file = "tornado-6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68"}, - {file = "tornado-6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085"}, - {file = "tornado-6.1-cp36-cp36m-win32.whl", hash = "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575"}, - {file = "tornado-6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795"}, - {file = "tornado-6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d"}, - {file = "tornado-6.1-cp37-cp37m-win32.whl", hash = "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df"}, - {file = "tornado-6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37"}, - {file = "tornado-6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f"}, - {file = "tornado-6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6"}, - {file = "tornado-6.1-cp38-cp38-win32.whl", hash = "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326"}, - {file = "tornado-6.1-cp38-cp38-win_amd64.whl", hash = "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c"}, - {file = "tornado-6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0"}, - {file = "tornado-6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd"}, - {file = "tornado-6.1-cp39-cp39-win32.whl", hash = "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c"}, - {file = "tornado-6.1-cp39-cp39-win_amd64.whl", hash = "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4"}, - {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, -] typed-ast = [ {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, @@ -1934,8 +1831,8 @@ urllib3 = [ {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] virtualenv = [ - {file = "virtualenv-20.13.3-py2.py3-none-any.whl", hash = "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021"}, - {file = "virtualenv-20.13.3.tar.gz", hash = "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134"}, + {file = "virtualenv-20.14.0-py2.py3-none-any.whl", hash = "sha256:1e8588f35e8b42c6ec6841a13c5e88239de1e6e4e4cedfd3916b306dc826ec66"}, + {file = "virtualenv-20.14.0.tar.gz", hash = "sha256:8e5b402037287126e81ccde9432b95a8be5b19d36584f64957060a3488c11ca8"}, ] xdoctest = [ {file = "xdoctest-0.15.10-py3-none-any.whl", hash = "sha256:7666bd0511df59275dfe94ef94b0fde9654afd14f00bf88902fdc9bcee77d527"}, diff --git a/pyproject.toml b/pyproject.toml index 62bccbd..fd24d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.0.1" description = "Osparc Control" authors = ["Andrei Neagu "] license = "MIT" -readme = "README.rst" +readme = "README.md" homepage = "https://github.com/ITISFoundation/osparc-control" repository = "https://github.com/ITISFoundation/osparc-control" documentation = "https://osparc-control.readthedocs.io" @@ -30,8 +30,6 @@ safety = "^1.10.3" mypy = "^0.910" typeguard = "^2.13.2" xdoctest = {extras = ["colors"], version = "^0.15.10"} -sphinx = "^4.3.0" -sphinx-autobuild = ">=2021.3.14" pre-commit = "^2.15.0" flake8 = "^4.0.1" black = ">=21.10b0" @@ -43,7 +41,6 @@ pep8-naming = "^0.12.1" darglint = "^1.8.1" reorder-python-imports = "^2.6.0" pre-commit-hooks = "^4.0.1" -sphinx-click = "^3.0.2" Pygments = "^2.10.0" pyupgrade = "^2.29.1" furo = ">=2021.11.12" From fe0c6b174480043dd9ddcfede59035a189e7288f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 10:43:04 +0200 Subject: [PATCH 76/86] added empty file --- docs/requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..e69de29 From 7ac3fa3d3255a4863f931b93251dd23230b43045 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 10:44:08 +0200 Subject: [PATCH 77/86] removed file --- docs/requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index e69de29..0000000 From f0349d58a8b4f980af8eaec081d6ad1e5057d267 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 10:47:21 +0200 Subject: [PATCH 78/86] updated docspage link --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd24d6d..eb675e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "MIT" readme = "README.md" homepage = "https://github.com/ITISFoundation/osparc-control" repository = "https://github.com/ITISFoundation/osparc-control" -documentation = "https://osparc-control.readthedocs.io" +documentation = "https://itisfoundation.github.io/osparc-control/" classifiers = [ "Development Status :: 3 - Alpha", ] From dad8dd2c2a9c16c078de760613e025bd884386a9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 10:54:56 +0200 Subject: [PATCH 79/86] removed unused --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb675e9..9672a3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,6 @@ pyupgrade = "^2.29.1" furo = ">=2021.11.12" rope = "^0.23.0" -[tool.poetry.scripts] -osparc-control = "osparc_control.__main__:main" [tool.coverage.paths] source = ["src", "*/site-packages"] From 4b5a4807ed25055140b3657980751bb99112b05e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 13:36:16 +0200 Subject: [PATCH 80/86] updated docs links and purged unused badge --- CONTRIBUTING.md | 2 +- README.md | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6dc16c1..fadf4c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,7 @@ This will allow a chance to talk it over with the owners and validate your appro [mit license]: https://opensource.org/licenses/MIT [source code]: https://github.com/ITISFoundation/osparc-control -[documentation]: https://osparc-control.readthedocs.io/ +[documentation]: https://itisfoundation.github.io/osparc-control/ [issue tracker]: https://github.com/ITISFoundation/osparc-control/issues [code of conduct]: CODE_OF_CONDUCT.rst [pull request]: https://github.com/ITISFoundation/osparc-control/pulls diff --git a/README.md b/README.md index 2276b16..e867d29 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ [![PyPI](https://img.shields.io/pypi/v/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Status](https://img.shields.io/pypi/status/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Python Version](https://img.shields.io/pypi/pyversions/osparc-control)](https://pypi.org/project/osparc-control) [![License](https://img.shields.io/pypi/l/osparc-control)](https://opensource.org/licenses/MIT) -[![Read the documentation at https://osparc-control.readthedocs.io/](https://img.shields.io/readthedocs/osparc-control/latest.svg?label=Read%20the%20Docs)](https://osparc-control.readthedocs.io/) [![Tests](https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg)](https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests) - -[![Codecov](https://codecov.io/gh/ITISFoundation/osparc-control/branch/main/graph/badge.svg)](https://codecov.io/gh/ITISFoundation/osparc-control) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Tests](https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg)](https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests) [![Codecov](https://codecov.io/gh/ITISFoundation/osparc-control/branch/main/graph/badge.svg)](https://codecov.io/gh/ITISFoundation/osparc-control) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) --- @@ -99,4 +97,3 @@ This project was generated from [@cjolowicz]'s [Hypermodern Python Cookiecutter] [pip]: https://pip.pypa.io/ [contributor guide]: CONTRIBUTING.md [code of conduct]: CODE_OF_CONDUCT.md -[usage]: https://osparc-control.readthedocs.io/en/latest/usage.html From 49c69435d57fa8197d9f6ca6d90329395e1abcf1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 13:38:36 +0200 Subject: [PATCH 81/86] added docs links on readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e867d29..ba4af1e 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ You can install _Osparc Control_ via [pip] from [PyPI]: pip install osparc-control ``` +## Documentation + +Read docs at https://itisfoundation.github.io/osparc-control + ## Examples To run below examples either clone the repo or copy code from the snippets From d9da53222c6ba6971d4f13b5331046d72f1b7496 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 13:49:57 +0200 Subject: [PATCH 82/86] added some more general usage fixtures --- tests/test_examples.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 597fcc0..84b5a38 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,5 +1,4 @@ import itertools -import sys from pathlib import Path from subprocess import PIPE # noqa: S404 from subprocess import Popen # noqa: S404 @@ -9,22 +8,20 @@ import pytest -HERE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - # FIXTURES @pytest.fixture -def path_1_simple() -> Path: - path = (HERE / ".." / "examples" / "1_simple").resolve() +def example_1_simple_path(examples_path: Path) -> Path: + path = (examples_path / "1_simple").resolve() assert path.exists() return path @pytest.fixture -def path_2_base_time_add() -> Path: - path = (HERE / ".." / "examples" / "2_base_time_add").resolve() +def example_2_base_time_add_path(examples_path: Path) -> Path: + path = (examples_path / "2_base_time_add").resolve() assert path.exists() return path @@ -64,15 +61,15 @@ def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: # TESTS -def test_example_1_simple_runs(path_1_simple: Path) -> None: - replier_path = path_1_simple / "replier.py" - requester_path = path_1_simple / "requester.py" +def test_example_1_simple_runs(example_1_simple_path: Path) -> None: + replier_path = example_1_simple_path / "replier.py" + requester_path = example_1_simple_path / "requester.py" assert_run_in_parallel([replier_path, requester_path]) -def test_example_2_base_time_add_runs(path_2_base_time_add: Path) -> None: - controller_path = path_2_base_time_add / "controller.py" - solver_path = path_2_base_time_add / "solver.py" +def test_example_2_base_time_add_runs(example_2_base_time_add_path: Path) -> None: + controller_path = example_2_base_time_add_path / "controller.py" + solver_path = example_2_base_time_add_path / "solver.py" assert_run_in_parallel([controller_path, solver_path]) From 9fbadc37f05640ced669f540c6c57dbcaed5b6ef Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 13:51:23 +0200 Subject: [PATCH 83/86] refacored test structure --- tests/test_core.py | 245 ---------------------------------------- tests/test_examples.py | 75 ------------ tests/test_models.py | 128 --------------------- tests/test_transport.py | 80 ------------- 4 files changed, 528 deletions(-) delete mode 100644 tests/test_core.py delete mode 100644 tests/test_examples.py delete mode 100644 tests/test_models.py delete mode 100644 tests/test_transport.py diff --git a/tests/test_core.py b/tests/test_core.py deleted file mode 100644 index 8e1a2e5..0000000 --- a/tests/test_core.py +++ /dev/null @@ -1,245 +0,0 @@ -import random -import time -from typing import Iterable -from typing import List - -import pytest -from pydantic import ValidationError - -import osparc_control -from osparc_control.core import PairedTransmitter -from osparc_control.errors import CommandConfirmationTimeoutError -from osparc_control.errors import CommandNotAcceptedError -from osparc_control.models import CommandManifest -from osparc_control.models import CommandParameter -from osparc_control.models import CommandType - -WAIT_FOR_DELIVERY = 0.1 - -ALL_COMMAND_TYPES: List[CommandType] = [ - CommandType.WITH_DELAYED_REPLY, - CommandType.WITH_DELAYED_REPLY, - CommandType.WITHOUT_REPLY, -] - -# UTILS - - -def _get_paired_transmitter( - *, local_port: int, remote_port: int, exposed_commands: List[CommandManifest] -) -> PairedTransmitter: - return PairedTransmitter( - remote_host="localhost", - exposed_commands=exposed_commands, - remote_port=remote_port, - listen_port=local_port, - ) - - -# FIXTURES - - -@pytest.fixture -def paired_transmitter_a() -> Iterable[PairedTransmitter]: - paired_transmitter = _get_paired_transmitter( - local_port=1234, remote_port=1235, exposed_commands=[] - ) - paired_transmitter.start_background_sync() - yield paired_transmitter - paired_transmitter.stop_background_sync() - - -@pytest.fixture -def mainfest_b() -> List[CommandManifest]: - add_numbers = CommandManifest( - action="add_numbers", - description="adds two numbers", - params=[ - CommandParameter(name="a", description="param to add"), - CommandParameter(name="b", description="param to add"), - ], - command_type=CommandType.WITH_DELAYED_REPLY, - ) - - get_random = CommandManifest( - action="get_random", - description="returns a random number", - params=[], - command_type=CommandType.WITH_IMMEDIATE_REPLY, - ) - - greet_user = CommandManifest( - action="greet_user", - description="prints the status of the solver", - params=[CommandParameter(name="name", description="name to greet")], - command_type=CommandType.WITHOUT_REPLY, - ) - - return [add_numbers, get_random, greet_user] - - -@pytest.fixture -def paired_transmitter_b( - mainfest_b: List[CommandManifest], -) -> Iterable[PairedTransmitter]: - paired_transmitter = _get_paired_transmitter( - local_port=1235, remote_port=1234, exposed_commands=mainfest_b - ) - paired_transmitter.start_background_sync() - yield paired_transmitter - paired_transmitter.stop_background_sync() - - -@pytest.fixture -def mock_wait_for_received() -> Iterable[None]: - previous = osparc_control.core.WAIT_FOR_RECEIVED_S - osparc_control.core.WAIT_FOR_RECEIVED_S = 0.01 - yield - osparc_control.core.WAIT_FOR_RECEIVED_S = previous - - -# TESTS - - -def test_context_manager(mainfest_b: List[CommandManifest]) -> None: - with _get_paired_transmitter( - local_port=1235, remote_port=1234, exposed_commands=mainfest_b - ): - pass - - -def test_request_with_delayed_reply( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - # SIDE A - request_id = paired_transmitter_a.request_with_delayed_reply( - "add_numbers", params={"a": 10, "b": 13.3} - ) - - # SIDE B - wait_for_requests = True - while wait_for_requests: - for command in paired_transmitter_b.get_incoming_requests(): - assert command.action == "add_numbers" - paired_transmitter_b.reply_to_command( - request_id=command.request_id, payload=sum(command.params.values()) - ) - wait_for_requests = False - - # SIDE A - time.sleep(WAIT_FOR_DELIVERY) - - has_result, result = paired_transmitter_a.check_for_reply(request_id=request_id) - assert has_result is True - assert result is not None - - -def test_request_with_immediate_reply( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - def _worker_b() -> None: - count = 1 - wait_for_requests = True - while wait_for_requests: - for command in paired_transmitter_b.get_incoming_requests(): - assert command.action == "get_random" - paired_transmitter_b.reply_to_command( - request_id=command.request_id, - payload=random.randint(1, 1000), # noqa: S311 - ) - count += 1 - - wait_for_requests = count > 2 - - from threading import Thread - - thread = Thread(target=_worker_b, daemon=True) - thread.start() - - random_integer = paired_transmitter_a.request_with_immediate_reply( - "get_random", timeout=1.0 - ) - assert type(random_integer) == int - assert random_integer - assert 1 <= random_integer <= 1000 - - no_reply_in_time = paired_transmitter_a.request_with_immediate_reply( - "get_random", timeout=0.001 - ) - assert no_reply_in_time is None - - thread.join() - - -def test_request_without_reply( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - # SIDE A - - paired_transmitter_a.request_without_reply("greet_user", params={"name": "tester"}) - expected_message = "hello tester" - - # SIDE B - wait_for_requests = True - while wait_for_requests: - for command in paired_transmitter_b.get_incoming_requests(): - assert command.action == "greet_user" - message = f"hello {command.params['name']}" - assert message == expected_message - wait_for_requests = False - - -@pytest.mark.parametrize("command_type", ALL_COMMAND_TYPES) -def test_no_same_action_command_in_exposed_commands(command_type: CommandType) -> None: - test_command_manifest = CommandManifest( - action="test", description="test", params=[], command_type=command_type - ) - - with pytest.raises(ValueError): - _get_paired_transmitter( - local_port=100, - remote_port=100, - exposed_commands=[test_command_manifest, test_command_manifest], - ) - - -def test_no_registered_command( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - with pytest.raises(CommandNotAcceptedError): - paired_transmitter_a.request_without_reply("command_not_defined") - - -def test_wrong_command_type( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - with pytest.raises(CommandNotAcceptedError): - paired_transmitter_a.request_without_reply("add_numbers") - - -def test_command_params_mismatch( - paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter -) -> None: - with pytest.raises(CommandNotAcceptedError): - paired_transmitter_a.request_without_reply("add_numbers", {"nope": 123}) - - -def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: - paired_transmitter = _get_paired_transmitter( - local_port=8263, remote_port=8263, exposed_commands=[] - ) - paired_transmitter.start_background_sync() - with pytest.raises(CommandConfirmationTimeoutError): - paired_transmitter.request_without_reply( - "no_remote_side_for_command", {"nope": 123} - ) - - -def test_paired_transmitter_validation() -> None: - with pytest.raises(ValidationError): - PairedTransmitter( - remote_host="localhost", - exposed_commands=[1], # type: ignore - remote_port=1, - listen_port=2, - ) diff --git a/tests/test_examples.py b/tests/test_examples.py deleted file mode 100644 index 84b5a38..0000000 --- a/tests/test_examples.py +++ /dev/null @@ -1,75 +0,0 @@ -import itertools -from pathlib import Path -from subprocess import PIPE # noqa: S404 -from subprocess import Popen # noqa: S404 -from subprocess import STDOUT # noqa: S404 -from time import sleep -from typing import List - -import pytest - - -# FIXTURES - - -@pytest.fixture -def example_1_simple_path(examples_path: Path) -> Path: - path = (examples_path / "1_simple").resolve() - assert path.exists() - return path - - -@pytest.fixture -def example_2_base_time_add_path(examples_path: Path) -> Path: - path = (examples_path / "2_base_time_add").resolve() - assert path.exists() - return path - - -# UTILS - - -def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: - # check all exist before running - for script_to_run in scrips_to_run: - assert script_to_run.exists() - - for permutation in itertools.permutations(scrips_to_run): - # run provided scripts - print(f"Running permutation {permutation}") - processes = [ - Popen( # noqa: S607 - ["python", f"{x}"], shell=True, stdout=PIPE, stderr=STDOUT # noqa: S602 - ) - for x in permutation - ] - - # wait for processes to finish - continue_running = True - while continue_running: - continue_running = all( - [process.returncode is not None for process in processes] - ) - sleep(0.3) - - # ensure all processes finished successfully - for process in processes: - stdout, _ = process.communicate() - assert process.returncode == 0, stdout.decode() - - -# TESTS - - -def test_example_1_simple_runs(example_1_simple_path: Path) -> None: - replier_path = example_1_simple_path / "replier.py" - requester_path = example_1_simple_path / "requester.py" - - assert_run_in_parallel([replier_path, requester_path]) - - -def test_example_2_base_time_add_runs(example_2_base_time_add_path: Path) -> None: - controller_path = example_2_base_time_add_path / "controller.py" - solver_path = example_2_base_time_add_path / "solver.py" - - assert_run_in_parallel([controller_path, solver_path]) diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 49ea1ff..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,128 +0,0 @@ -import json -from typing import Any -from typing import Dict -from typing import List -from typing import Optional - -import pytest -import umsgpack # type: ignore -from pydantic import ValidationError - -from osparc_control.models import CommandManifest -from osparc_control.models import CommandParameter -from osparc_control.models import CommandReceived -from osparc_control.models import CommandReply -from osparc_control.models import CommandRequest -from osparc_control.models import CommandType - - -@pytest.fixture -def request_id() -> str: - return "unique_id" - - -PARAMS: List[Optional[List[CommandParameter]]] = [ - None, - [], - [CommandParameter(name="first_arg", description="the first arg description")], -] - - -@pytest.mark.parametrize("params", PARAMS) -def test_command_manifest(params: Optional[List[CommandParameter]]) -> None: - for command_type in CommandType: - assert CommandManifest( - action="test", - description="some test action", - command_type=command_type, - params=[] if params is None else params, - ) - - -@pytest.mark.parametrize("params", PARAMS) -def test_command(request_id: str, params: Optional[List[CommandParameter]]) -> None: - request_params: Dict[str, Any] = ( - {} if params is None else {x.name: None for x in params} - ) - manifest = CommandManifest( - action="test", - description="some test action", - command_type=CommandType.WITHOUT_REPLY, - params=[] if params is None else params, - ) - - assert CommandRequest( - request_id=request_id, - action=manifest.action, - command_type=manifest.command_type, - params=request_params, - ) - - -@pytest.mark.parametrize("params", PARAMS) -def test_msgpack_serialization_deserialization( - request_id: str, params: Optional[List[CommandParameter]] -) -> None: - - request_params: Dict[str, Any] = ( - {} if params is None else {x.name: None for x in params} - ) - manifest = CommandManifest( - action="test", - description="some test action", - command_type=CommandType.WITH_IMMEDIATE_REPLY, - params=[] if params is None else params, - ) - - command_request = CommandRequest( - request_id=request_id, - action=manifest.action, - command_type=manifest.command_type, - params=request_params, - ) - - assert command_request == CommandRequest.from_bytes(command_request.to_bytes()) - - assert command_request.to_bytes() == umsgpack.packb( - json.loads(command_request.json()) - ) - - -@pytest.mark.parametrize("payload", [None, "a_string", 1, 1.0, b"some_bytes"]) -def test_command_reply_payloads_serialization_deserialization( - request_id: str, payload: Any -) -> None: - command_reply = CommandReply(reply_id=request_id, payload=payload) - assert command_reply - assert command_reply == CommandReply.from_bytes(command_reply.to_bytes()) - - -def test_command_accepted_ok(request_id: str) -> None: - assert CommandReceived(request_id=request_id, accepted=True, error_message=None) - assert CommandReceived( - request_id=request_id, accepted=False, error_message="some error" - ) - - -def test_command_accepted_fails(request_id: str) -> None: - with pytest.raises(ValidationError): - assert CommandReceived( - request_id=request_id, accepted=False, error_message=None - ) - with pytest.raises(ValidationError): - assert CommandReceived( - request_id=request_id, accepted=True, error_message="some error" - ) - - -def test_duplicate_command_parameter_name_in_manifest() -> None: - with pytest.raises(ValidationError): - CommandManifest( - action="test", - description="with invalid paramters", - params=[ - CommandParameter(name="a", description="ok"), - CommandParameter(name="a", description="not allowed same name"), - ], - command_type=CommandType.WITH_DELAYED_REPLY, - ) diff --git a/tests/test_transport.py b/tests/test_transport.py deleted file mode 100644 index b5b8956..0000000 --- a/tests/test_transport.py +++ /dev/null @@ -1,80 +0,0 @@ -from typing import Iterable -from typing import Optional -from typing import Type - -import pytest -from _pytest.fixtures import SubRequest - -from osparc_control.transport.base_transport import BaseTransport -from osparc_control.transport.base_transport import SenderReceiverPair -from osparc_control.transport.in_memory import InMemoryTransport -from osparc_control.transport.zeromq import ZeroMQTransport - -# UTILS - - -def _payload_generator(start: int, stop: int) -> Iterable[bytes]: - assert start < stop - for k in range(start, stop): - yield f"test{k}".encode() - - -# TESTS - - -@pytest.fixture(params=[InMemoryTransport, ZeroMQTransport]) -def transport_class(request: SubRequest) -> Type[BaseTransport]: - return request.param # type: ignore - - -@pytest.fixture -def sender_receiver_pair( - transport_class: Type[BaseTransport], -) -> Iterable[SenderReceiverPair]: - sender: Optional[BaseTransport] = None - receiver: Optional[BaseTransport] = None - - if transport_class == InMemoryTransport: - sender = transport_class("A", "B") # type: ignore - receiver = transport_class("B", "A") # type: ignore - - if transport_class == ZeroMQTransport: - port = 1111 - sender = transport_class( # type: ignore - listen_port=port, remote_host="localhost", remote_port=port - ) - receiver = transport_class( # type: ignore - listen_port=port, remote_host="localhost", remote_port=port - ) - - assert sender - assert receiver - sender_receiver_pair = SenderReceiverPair(sender=sender, receiver=receiver) - - sender_receiver_pair.sender_init() - sender_receiver_pair.receiver_init() - - yield sender_receiver_pair - - sender_receiver_pair.sender_cleanup() - sender_receiver_pair.receiver_cleanup() - - -def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair) -> None: - for message in _payload_generator(1, 10): - print("sending", message) - sender_receiver_pair.send_bytes(message) - - for expected_message in _payload_generator(1, 10): - received_message: Optional[bytes] = sender_receiver_pair.receive_bytes() - print("received", received_message) - assert received_message == expected_message - - -def test_receive_nothing(sender_receiver_pair: SenderReceiverPair) -> None: - assert sender_receiver_pair.receive_bytes() is None - - -def test_receive_returns_none_if_no_message_available() -> None: - receiver = InMemoryTransport("B", "A") - assert receiver.receive_bytes() is None From 08c698bb0a6be0b209280df3fc5ce06850792582 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 13:53:29 +0200 Subject: [PATCH 84/86] new test structure --- tests/conftest.py | 17 +++ tests/examples/test_all.py | 75 +++++++++++ tests/unit/test_core.py | 245 +++++++++++++++++++++++++++++++++++ tests/unit/test_models.py | 128 ++++++++++++++++++ tests/unit/test_transport.py | 80 ++++++++++++ 5 files changed, 545 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/examples/test_all.py create mode 100644 tests/unit/test_core.py create mode 100644 tests/unit/test_models.py create mode 100644 tests/unit/test_transport.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ffba6ad --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def repo_folder() -> Path: + here = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + return (here / "..").resolve() + + +@pytest.fixture +def examples_path(repo_folder: Path) -> Path: + path = repo_folder / "examples" + assert path.exists() + return path diff --git a/tests/examples/test_all.py b/tests/examples/test_all.py new file mode 100644 index 0000000..84b5a38 --- /dev/null +++ b/tests/examples/test_all.py @@ -0,0 +1,75 @@ +import itertools +from pathlib import Path +from subprocess import PIPE # noqa: S404 +from subprocess import Popen # noqa: S404 +from subprocess import STDOUT # noqa: S404 +from time import sleep +from typing import List + +import pytest + + +# FIXTURES + + +@pytest.fixture +def example_1_simple_path(examples_path: Path) -> Path: + path = (examples_path / "1_simple").resolve() + assert path.exists() + return path + + +@pytest.fixture +def example_2_base_time_add_path(examples_path: Path) -> Path: + path = (examples_path / "2_base_time_add").resolve() + assert path.exists() + return path + + +# UTILS + + +def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: + # check all exist before running + for script_to_run in scrips_to_run: + assert script_to_run.exists() + + for permutation in itertools.permutations(scrips_to_run): + # run provided scripts + print(f"Running permutation {permutation}") + processes = [ + Popen( # noqa: S607 + ["python", f"{x}"], shell=True, stdout=PIPE, stderr=STDOUT # noqa: S602 + ) + for x in permutation + ] + + # wait for processes to finish + continue_running = True + while continue_running: + continue_running = all( + [process.returncode is not None for process in processes] + ) + sleep(0.3) + + # ensure all processes finished successfully + for process in processes: + stdout, _ = process.communicate() + assert process.returncode == 0, stdout.decode() + + +# TESTS + + +def test_example_1_simple_runs(example_1_simple_path: Path) -> None: + replier_path = example_1_simple_path / "replier.py" + requester_path = example_1_simple_path / "requester.py" + + assert_run_in_parallel([replier_path, requester_path]) + + +def test_example_2_base_time_add_runs(example_2_base_time_add_path: Path) -> None: + controller_path = example_2_base_time_add_path / "controller.py" + solver_path = example_2_base_time_add_path / "solver.py" + + assert_run_in_parallel([controller_path, solver_path]) diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py new file mode 100644 index 0000000..8e1a2e5 --- /dev/null +++ b/tests/unit/test_core.py @@ -0,0 +1,245 @@ +import random +import time +from typing import Iterable +from typing import List + +import pytest +from pydantic import ValidationError + +import osparc_control +from osparc_control.core import PairedTransmitter +from osparc_control.errors import CommandConfirmationTimeoutError +from osparc_control.errors import CommandNotAcceptedError +from osparc_control.models import CommandManifest +from osparc_control.models import CommandParameter +from osparc_control.models import CommandType + +WAIT_FOR_DELIVERY = 0.1 + +ALL_COMMAND_TYPES: List[CommandType] = [ + CommandType.WITH_DELAYED_REPLY, + CommandType.WITH_DELAYED_REPLY, + CommandType.WITHOUT_REPLY, +] + +# UTILS + + +def _get_paired_transmitter( + *, local_port: int, remote_port: int, exposed_commands: List[CommandManifest] +) -> PairedTransmitter: + return PairedTransmitter( + remote_host="localhost", + exposed_commands=exposed_commands, + remote_port=remote_port, + listen_port=local_port, + ) + + +# FIXTURES + + +@pytest.fixture +def paired_transmitter_a() -> Iterable[PairedTransmitter]: + paired_transmitter = _get_paired_transmitter( + local_port=1234, remote_port=1235, exposed_commands=[] + ) + paired_transmitter.start_background_sync() + yield paired_transmitter + paired_transmitter.stop_background_sync() + + +@pytest.fixture +def mainfest_b() -> List[CommandManifest]: + add_numbers = CommandManifest( + action="add_numbers", + description="adds two numbers", + params=[ + CommandParameter(name="a", description="param to add"), + CommandParameter(name="b", description="param to add"), + ], + command_type=CommandType.WITH_DELAYED_REPLY, + ) + + get_random = CommandManifest( + action="get_random", + description="returns a random number", + params=[], + command_type=CommandType.WITH_IMMEDIATE_REPLY, + ) + + greet_user = CommandManifest( + action="greet_user", + description="prints the status of the solver", + params=[CommandParameter(name="name", description="name to greet")], + command_type=CommandType.WITHOUT_REPLY, + ) + + return [add_numbers, get_random, greet_user] + + +@pytest.fixture +def paired_transmitter_b( + mainfest_b: List[CommandManifest], +) -> Iterable[PairedTransmitter]: + paired_transmitter = _get_paired_transmitter( + local_port=1235, remote_port=1234, exposed_commands=mainfest_b + ) + paired_transmitter.start_background_sync() + yield paired_transmitter + paired_transmitter.stop_background_sync() + + +@pytest.fixture +def mock_wait_for_received() -> Iterable[None]: + previous = osparc_control.core.WAIT_FOR_RECEIVED_S + osparc_control.core.WAIT_FOR_RECEIVED_S = 0.01 + yield + osparc_control.core.WAIT_FOR_RECEIVED_S = previous + + +# TESTS + + +def test_context_manager(mainfest_b: List[CommandManifest]) -> None: + with _get_paired_transmitter( + local_port=1235, remote_port=1234, exposed_commands=mainfest_b + ): + pass + + +def test_request_with_delayed_reply( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + # SIDE A + request_id = paired_transmitter_a.request_with_delayed_reply( + "add_numbers", params={"a": 10, "b": 13.3} + ) + + # SIDE B + wait_for_requests = True + while wait_for_requests: + for command in paired_transmitter_b.get_incoming_requests(): + assert command.action == "add_numbers" + paired_transmitter_b.reply_to_command( + request_id=command.request_id, payload=sum(command.params.values()) + ) + wait_for_requests = False + + # SIDE A + time.sleep(WAIT_FOR_DELIVERY) + + has_result, result = paired_transmitter_a.check_for_reply(request_id=request_id) + assert has_result is True + assert result is not None + + +def test_request_with_immediate_reply( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + def _worker_b() -> None: + count = 1 + wait_for_requests = True + while wait_for_requests: + for command in paired_transmitter_b.get_incoming_requests(): + assert command.action == "get_random" + paired_transmitter_b.reply_to_command( + request_id=command.request_id, + payload=random.randint(1, 1000), # noqa: S311 + ) + count += 1 + + wait_for_requests = count > 2 + + from threading import Thread + + thread = Thread(target=_worker_b, daemon=True) + thread.start() + + random_integer = paired_transmitter_a.request_with_immediate_reply( + "get_random", timeout=1.0 + ) + assert type(random_integer) == int + assert random_integer + assert 1 <= random_integer <= 1000 + + no_reply_in_time = paired_transmitter_a.request_with_immediate_reply( + "get_random", timeout=0.001 + ) + assert no_reply_in_time is None + + thread.join() + + +def test_request_without_reply( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + # SIDE A + + paired_transmitter_a.request_without_reply("greet_user", params={"name": "tester"}) + expected_message = "hello tester" + + # SIDE B + wait_for_requests = True + while wait_for_requests: + for command in paired_transmitter_b.get_incoming_requests(): + assert command.action == "greet_user" + message = f"hello {command.params['name']}" + assert message == expected_message + wait_for_requests = False + + +@pytest.mark.parametrize("command_type", ALL_COMMAND_TYPES) +def test_no_same_action_command_in_exposed_commands(command_type: CommandType) -> None: + test_command_manifest = CommandManifest( + action="test", description="test", params=[], command_type=command_type + ) + + with pytest.raises(ValueError): + _get_paired_transmitter( + local_port=100, + remote_port=100, + exposed_commands=[test_command_manifest, test_command_manifest], + ) + + +def test_no_registered_command( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + with pytest.raises(CommandNotAcceptedError): + paired_transmitter_a.request_without_reply("command_not_defined") + + +def test_wrong_command_type( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + with pytest.raises(CommandNotAcceptedError): + paired_transmitter_a.request_without_reply("add_numbers") + + +def test_command_params_mismatch( + paired_transmitter_a: PairedTransmitter, paired_transmitter_b: PairedTransmitter +) -> None: + with pytest.raises(CommandNotAcceptedError): + paired_transmitter_a.request_without_reply("add_numbers", {"nope": 123}) + + +def test_side_b_does_not_reply_in_time(mock_wait_for_received: None) -> None: + paired_transmitter = _get_paired_transmitter( + local_port=8263, remote_port=8263, exposed_commands=[] + ) + paired_transmitter.start_background_sync() + with pytest.raises(CommandConfirmationTimeoutError): + paired_transmitter.request_without_reply( + "no_remote_side_for_command", {"nope": 123} + ) + + +def test_paired_transmitter_validation() -> None: + with pytest.raises(ValidationError): + PairedTransmitter( + remote_host="localhost", + exposed_commands=[1], # type: ignore + remote_port=1, + listen_port=2, + ) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py new file mode 100644 index 0000000..49ea1ff --- /dev/null +++ b/tests/unit/test_models.py @@ -0,0 +1,128 @@ +import json +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + +import pytest +import umsgpack # type: ignore +from pydantic import ValidationError + +from osparc_control.models import CommandManifest +from osparc_control.models import CommandParameter +from osparc_control.models import CommandReceived +from osparc_control.models import CommandReply +from osparc_control.models import CommandRequest +from osparc_control.models import CommandType + + +@pytest.fixture +def request_id() -> str: + return "unique_id" + + +PARAMS: List[Optional[List[CommandParameter]]] = [ + None, + [], + [CommandParameter(name="first_arg", description="the first arg description")], +] + + +@pytest.mark.parametrize("params", PARAMS) +def test_command_manifest(params: Optional[List[CommandParameter]]) -> None: + for command_type in CommandType: + assert CommandManifest( + action="test", + description="some test action", + command_type=command_type, + params=[] if params is None else params, + ) + + +@pytest.mark.parametrize("params", PARAMS) +def test_command(request_id: str, params: Optional[List[CommandParameter]]) -> None: + request_params: Dict[str, Any] = ( + {} if params is None else {x.name: None for x in params} + ) + manifest = CommandManifest( + action="test", + description="some test action", + command_type=CommandType.WITHOUT_REPLY, + params=[] if params is None else params, + ) + + assert CommandRequest( + request_id=request_id, + action=manifest.action, + command_type=manifest.command_type, + params=request_params, + ) + + +@pytest.mark.parametrize("params", PARAMS) +def test_msgpack_serialization_deserialization( + request_id: str, params: Optional[List[CommandParameter]] +) -> None: + + request_params: Dict[str, Any] = ( + {} if params is None else {x.name: None for x in params} + ) + manifest = CommandManifest( + action="test", + description="some test action", + command_type=CommandType.WITH_IMMEDIATE_REPLY, + params=[] if params is None else params, + ) + + command_request = CommandRequest( + request_id=request_id, + action=manifest.action, + command_type=manifest.command_type, + params=request_params, + ) + + assert command_request == CommandRequest.from_bytes(command_request.to_bytes()) + + assert command_request.to_bytes() == umsgpack.packb( + json.loads(command_request.json()) + ) + + +@pytest.mark.parametrize("payload", [None, "a_string", 1, 1.0, b"some_bytes"]) +def test_command_reply_payloads_serialization_deserialization( + request_id: str, payload: Any +) -> None: + command_reply = CommandReply(reply_id=request_id, payload=payload) + assert command_reply + assert command_reply == CommandReply.from_bytes(command_reply.to_bytes()) + + +def test_command_accepted_ok(request_id: str) -> None: + assert CommandReceived(request_id=request_id, accepted=True, error_message=None) + assert CommandReceived( + request_id=request_id, accepted=False, error_message="some error" + ) + + +def test_command_accepted_fails(request_id: str) -> None: + with pytest.raises(ValidationError): + assert CommandReceived( + request_id=request_id, accepted=False, error_message=None + ) + with pytest.raises(ValidationError): + assert CommandReceived( + request_id=request_id, accepted=True, error_message="some error" + ) + + +def test_duplicate_command_parameter_name_in_manifest() -> None: + with pytest.raises(ValidationError): + CommandManifest( + action="test", + description="with invalid paramters", + params=[ + CommandParameter(name="a", description="ok"), + CommandParameter(name="a", description="not allowed same name"), + ], + command_type=CommandType.WITH_DELAYED_REPLY, + ) diff --git a/tests/unit/test_transport.py b/tests/unit/test_transport.py new file mode 100644 index 0000000..b5b8956 --- /dev/null +++ b/tests/unit/test_transport.py @@ -0,0 +1,80 @@ +from typing import Iterable +from typing import Optional +from typing import Type + +import pytest +from _pytest.fixtures import SubRequest + +from osparc_control.transport.base_transport import BaseTransport +from osparc_control.transport.base_transport import SenderReceiverPair +from osparc_control.transport.in_memory import InMemoryTransport +from osparc_control.transport.zeromq import ZeroMQTransport + +# UTILS + + +def _payload_generator(start: int, stop: int) -> Iterable[bytes]: + assert start < stop + for k in range(start, stop): + yield f"test{k}".encode() + + +# TESTS + + +@pytest.fixture(params=[InMemoryTransport, ZeroMQTransport]) +def transport_class(request: SubRequest) -> Type[BaseTransport]: + return request.param # type: ignore + + +@pytest.fixture +def sender_receiver_pair( + transport_class: Type[BaseTransport], +) -> Iterable[SenderReceiverPair]: + sender: Optional[BaseTransport] = None + receiver: Optional[BaseTransport] = None + + if transport_class == InMemoryTransport: + sender = transport_class("A", "B") # type: ignore + receiver = transport_class("B", "A") # type: ignore + + if transport_class == ZeroMQTransport: + port = 1111 + sender = transport_class( # type: ignore + listen_port=port, remote_host="localhost", remote_port=port + ) + receiver = transport_class( # type: ignore + listen_port=port, remote_host="localhost", remote_port=port + ) + + assert sender + assert receiver + sender_receiver_pair = SenderReceiverPair(sender=sender, receiver=receiver) + + sender_receiver_pair.sender_init() + sender_receiver_pair.receiver_init() + + yield sender_receiver_pair + + sender_receiver_pair.sender_cleanup() + sender_receiver_pair.receiver_cleanup() + + +def test_send_receive_single_thread(sender_receiver_pair: SenderReceiverPair) -> None: + for message in _payload_generator(1, 10): + print("sending", message) + sender_receiver_pair.send_bytes(message) + + for expected_message in _payload_generator(1, 10): + received_message: Optional[bytes] = sender_receiver_pair.receive_bytes() + print("received", received_message) + assert received_message == expected_message + + +def test_receive_nothing(sender_receiver_pair: SenderReceiverPair) -> None: + assert sender_receiver_pair.receive_bytes() is None + + +def test_receive_returns_none_if_no_message_available() -> None: + receiver = InMemoryTransport("B", "A") + assert receiver.receive_bytes() is None From ff83b6b91d713f918404f9c049920ab226ac98a0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 14:02:02 +0200 Subject: [PATCH 85/86] updated coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba4af1e..af498f5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI](https://img.shields.io/pypi/v/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Status](https://img.shields.io/pypi/status/osparc-control.svg)](https://pypi.org/project/osparc-control/) [![Python Version](https://img.shields.io/pypi/pyversions/osparc-control)](https://pypi.org/project/osparc-control) [![License](https://img.shields.io/pypi/l/osparc-control)](https://opensource.org/licenses/MIT) -[![Tests](https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg)](https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests) [![Codecov](https://codecov.io/gh/ITISFoundation/osparc-control/branch/main/graph/badge.svg)](https://codecov.io/gh/ITISFoundation/osparc-control) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Tests](https://github.com/ITISFoundation/osparc-control/workflows/Tests/badge.svg)](https://github.com/ITISFoundation/osparc-control/actions?workflow=Tests) [![codecov](https://codecov.io/gh/ITISFoundation/osparc-control/branch/master/graph/badge.svg?token=3P04fQlaEb)](https://codecov.io/gh/ITISFoundation/osparc-control) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) --- From 35a5dadb077c63be9db2a2eaa1caac4c6a2b0cd8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Apr 2022 16:08:35 +0200 Subject: [PATCH 86/86] add delay when booting services --- tests/examples/test_all.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/examples/test_all.py b/tests/examples/test_all.py index 84b5a38..e3636ba 100644 --- a/tests/examples/test_all.py +++ b/tests/examples/test_all.py @@ -9,6 +9,9 @@ import pytest +DELAY_PRCESS_START_S: float = 1.0 +WAIT_BETWEEN_CHECKS_S: float = 0.3 + # FIXTURES @@ -37,20 +40,21 @@ def assert_run_in_parallel(scrips_to_run: List[Path]) -> None: for permutation in itertools.permutations(scrips_to_run): # run provided scripts print(f"Running permutation {permutation}") - processes = [ - Popen( # noqa: S607 + processes = [] + for x in permutation: + process = Popen( # noqa: S607 ["python", f"{x}"], shell=True, stdout=PIPE, stderr=STDOUT # noqa: S602 ) - for x in permutation - ] + sleep(DELAY_PRCESS_START_S) + processes.append(process) # wait for processes to finish continue_running = True while continue_running: continue_running = all( - [process.returncode is not None for process in processes] + process.returncode is not None for process in processes ) - sleep(0.3) + sleep(WAIT_BETWEEN_CHECKS_S) # ensure all processes finished successfully for process in processes: