From 95cc9ff14f1724750234b57a3e4818f56b908a19 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 9 Oct 2020 16:40:21 +0200 Subject: [PATCH 01/42] Fix Python Grid issues (#75 and #77) --- libmuscle/python/libmuscle/mcp/message.py | 31 +++--------- .../python/libmuscle/mcp/test/test_message.py | 49 ++++++++++++++++--- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/libmuscle/python/libmuscle/mcp/message.py b/libmuscle/python/libmuscle/mcp/message.py index bdd25723..f5a6b8a8 100644 --- a/libmuscle/python/libmuscle/mcp/message.py +++ b/libmuscle/python/libmuscle/mcp/message.py @@ -1,5 +1,4 @@ from enum import IntEnum -import struct from typing import Any, cast, Optional import msgpack @@ -49,20 +48,6 @@ class ClosePort: def _encode_grid(grid: Grid) -> msgpack.ExtType: """Encodes a Grid object into the wire format. """ - item_size_map = { - 'int32': 4, - 'int64': 8, - 'float32': 4, - 'float64': 8, - 'bool': 1} - - format_map = { - 'int32': ' msgpack.ExtType: 'bool': ExtTypeId.GRID_BOOL} array = grid.array - if array.flags.c_contiguous: - # indexes that differ in the last place are adjacent - order = 'la' - elif array.flags.f_contiguous: + if array.flags.f_contiguous: # indexes that differ in the first place are adjacent order = 'fa' + else: + # indexes that differ in the last place are adjacent + order = 'la' # dtype is a bit weird, but this seems to be consistent if isinstance(array.dtype, np.dtype): @@ -84,14 +69,10 @@ def _encode_grid(grid: Grid) -> msgpack.ExtType: else: array_type = str(np.dtype(array_type)) - if array_type not in item_size_map: + if array_type not in ext_type_map: raise RuntimeError('Unsupported array data type') - item_size = item_size_map[array_type] - fmt = format_map[array_type] - buf = bytearray(array.size * item_size) - for i in range(array.size): - struct.pack_into(fmt, buf, i * item_size, array.item(i)) + buf = array.tobytes(order='A') # array_type is redundant, but useful metadata. grid_dict = { diff --git a/libmuscle/python/libmuscle/mcp/test/test_message.py b/libmuscle/python/libmuscle/mcp/test/test_message.py index 1a968068..30dad34a 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_message.py +++ b/libmuscle/python/libmuscle/mcp/test/test_message.py @@ -121,15 +121,49 @@ def test_grid_roundtrip() -> None: timestamp = 10.0 next_timestamp = 11.0 + for order in ('C', 'F'): + array = np.array( + [[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[7.0, 8.0, 9.0], + [10.0, 11.0, 12.0]]], np.float64, order=order) + + assert array[0, 0, 0] == 1.0 + + grid = Grid(array, ['x', 'y', 'z']) + msg = Message(sender, receiver, None, timestamp, next_timestamp, + Settings(), grid) + + wire_data = msg.encoded() + msg_out = Message.from_bytes(wire_data) + + assert isinstance(msg_out.data, Grid) + grid_out = msg_out.data + assert grid_out.indexes == ['x', 'y', 'z'] + assert isinstance(grid_out.array, np.ndarray) + assert grid_out.array.dtype == np.float64 + assert grid_out.array.shape == (2, 2, 3) + assert grid_out.array.size == 12 + assert grid_out.array[1, 0, 1] == 8.0 + assert grid_out.array[0, 0, 2] == 3.0 + + +def test_non_contiguous_grid_roundtrip() -> None: + sender = Reference('sender.port') + receiver = Reference('receiver.port') + timestamp = 10.0 + next_timestamp = 11.0 + array = np.array( - [[[1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]], - [[7.0, 8.0, 9.0], - [10.0, 11.0, 12.0]]], np.float64) + [[[1.0 + 0.125j, 2.0 + 0.25j, 3.0 + 0.375j], + [4.0 + 0.4375j, 5.0 + 0.5j, 6.0 + 0.625j]], + [[7.0 + 0.75j, 8.0 + 0.875j, 9.0 + 0.9375j], + [10.0 + 0.10j, 11.0 + 0.11j, 12.0 + 0.12j]]], np.complex64) - assert(array[0, 0, 0] == 1.0) + assert array.real[0, 0, 0] == 1.0 + assert array.imag[0, 0, 0] == 0.125 - grid = Grid(array, ['x', 'y', 'z']) + grid = Grid(array.real, ['a', 'b', 'c']) msg = Message(sender, receiver, None, timestamp, next_timestamp, Settings(), grid) @@ -138,9 +172,8 @@ def test_grid_roundtrip() -> None: assert isinstance(msg_out.data, Grid) grid_out = msg_out.data - assert grid_out.indexes == ['x', 'y', 'z'] assert isinstance(grid_out.array, np.ndarray) - assert grid_out.array.dtype == np.float64 + assert grid_out.array.dtype == np.float32 assert grid_out.array.shape == (2, 2, 3) assert grid_out.array.size == 12 assert grid_out.array[1, 0, 1] == 8.0 From bc85380d47c15adc2833429abf77216df65a4077 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 9 Oct 2020 16:48:11 +0200 Subject: [PATCH 02/42] Fix typo in documentation --- docs/source/installing.rst.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installing.rst.in b/docs/source/installing.rst.in index d57974c1..0560b3c9 100644 --- a/docs/source/installing.rst.in +++ b/docs/source/installing.rst.in @@ -109,7 +109,7 @@ The dependencies are: If your model uses any of these dependencies directly, then it's best to install that dependency on your system, either via the package manager or from source, and then link both your library and MUSCLE 3 to the dependency. (See below for -how t opoint the build to your installation.) This avoids having two different +how to point the build to your installation.) This avoids having two different versions around and active at the same time. Otherwise, it's easier to rely on the automatic installation. Note that the gRPC and Protobuf dependencies are exact; getting them to install and work correctly on all systems is From 6a596cec79729a353cf114bcdfb9f26fbe4d2273 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 6 Nov 2020 17:42:56 +0100 Subject: [PATCH 03/42] Add MUSCLE 3 paper to CITATION.cff and docs. Closes #76. --- CITATION.cff | 49 ++++++++++++++++++++++++++++++++++++++----- docs/source/index.rst | 15 +++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index d576d1a0..9b2c5304 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,18 +1,57 @@ # YAML 1.2 --- +cff-version: "1.1.0" + +title: "MUSCLE 3: The Multiscale Coupling Library and Environment" +doi: "10.5281/zenodo.3258864" +repository-code: "https://github.com/multiscale/muscle3" + +message: "If you use MUSCLE 3, please cite it using these metadata." authors: - family-names: Veen given-names: Lourens orcid: "https://orcid.org/0000-0002-6311-1168" -cff-version: "1.1.0" -doi: "10.5281/zenodo.3258864" + keywords: - multiscale - coupling - MUSCLE + license: "Apache-2.0" -message: "If you use MUSCLE 3, please cite it using these metadata." -repository-code: "https://github.com/multiscale/muscle3" -title: "MUSCLE 3: The Multiscale Coupling Library and Environment" + +references: + - type: conference-paper + doi: "10.1007/978-3-030-50433-5_33" + authors: + - family-names: Veen + given-names: Lourens E. + - family-names: Hoekstra + given-names: Alfons G. + title: Easing Multiscale Model Design and Coupling with MUSCLE 3 + year: 2020 + + collection-title: Lecture Notes in Computer Science + volume: 12142 + volume-title: Computational Science -- ICCS 2020 + editors: + - family-names: Krzhizhanovskaya + given-names: Valeria V. + - family-names: Závodszky + given-names: Gábor + - family-names: Lees + given-names: Michael H. + - family-names: Dongarra + given-names: Jack J. + - family-names: Sloot + given-names: Peter M. A. + - family-names: Brissos + given-names: Sèrgio + - family-names: Teixeira + given-names: João + start: 425 + end: 438 + publisher: + name: Springer International Publishing + city: Cham ... diff --git a/docs/source/index.rst b/docs/source/index.rst index d584d604..45291e2d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,6 +9,21 @@ Welcome to MUSCLE 3's documentation! MUSCLE 3 is the third incarnation of the Multiscale Coupling Library and Environment. +With MUSCLE 3, you can connect multiple simulation models together into a +multiscale simulation. Simulation models can be as simple as a single Python +file, or as complex as a combination of multiple separate simulation codes +written in C++ or Fortran, and running on an HPC machine. + +If you use MUSCLE 3 for scientific work, please `cite +the version of the MUSCLE 3 software you used +`_ and the following paper: + +Veen L.E., Hoekstra A.G. (2020) Easing Multiscale Model Design and Coupling +with MUSCLE 3. In: Krzhizhanovskaya V. et al. (eds) Computational Science – +ICCS 2020. ICCS 2020. Lecture Notes in Computer Science, vol 12142. Springer, +Cham. ``_ + + .. toctree:: :maxdepth: 4 :caption: Contents: From db0ce282a75d474d29d7b9e2eded80d62a761f90 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 8 Nov 2020 15:20:45 +0100 Subject: [PATCH 04/42] Lazy-allocate smaller zones for Data/DCR objects to save memory --- libmuscle/cpp/src/libmuscle/data.cpp | 1 - libmuscle/cpp/src/libmuscle/data.tpp | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp index c7b0245c..1aa46827 100644 --- a/libmuscle/cpp/src/libmuscle/data.cpp +++ b/libmuscle/cpp/src/libmuscle/data.cpp @@ -97,7 +97,6 @@ DataConstRef DataConstRef::grid_data_( DataConstRef::DataConstRef() : mp_zones_(new std::vector>()) { - mp_zones_->push_back(std::make_shared()); mp_obj_ = zone_alloc_(); mp_obj_->type = msgpack::type::NIL; } diff --git a/libmuscle/cpp/src/libmuscle/data.tpp b/libmuscle/cpp/src/libmuscle/data.tpp index 3276fb1b..c2a4d5ec 100644 --- a/libmuscle/cpp/src/libmuscle/data.tpp +++ b/libmuscle/cpp/src/libmuscle/data.tpp @@ -30,6 +30,8 @@ T DataConstRef::as() const { template T * DataConstRef::zone_alloc_(uint32_t size) { + if (mp_zones_->empty()) + mp_zones_->push_back(std::make_shared(24)); auto num_bytes = sizeof(T) * size; return static_cast((*mp_zones_)[0]->allocate_align( num_bytes, MSGPACK_ZONE_ALIGNOF(T))); From 9d289be316784c6fe85e505688014b4018edd620 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 13 Nov 2020 11:16:02 +0100 Subject: [PATCH 05/42] Only copy zones if needed when assigning a Data --- libmuscle/cpp/src/libmuscle/data.cpp | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp index 1aa46827..a69bd2fc 100644 --- a/libmuscle/cpp/src/libmuscle/data.cpp +++ b/libmuscle/cpp/src/libmuscle/data.cpp @@ -845,17 +845,23 @@ Data Data::byte_array(char const * buf, uint32_t size) { Data & Data::operator=(Data const & rhs) { if (mp_obj_ != rhs.mp_obj_) { *mp_obj_ = *rhs.mp_obj_; - - // We can't overwrite mp_zones_ here, because mp_obj_ is allocated on - // one of them, and we don't know which. So we just append, which is - // suboptimal because it may keep objects alive that are no longer - // reachable. Consider a separate shared_ptr to the zone that mp_obj_ - // is on (in a separate member), so that we can safely overwrite - // mp_zones_. - if (mp_zones_ != rhs.mp_zones_) - mp_zones_->insert(mp_zones_->end(), - rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend()); obj_cache_ = rhs.obj_cache_; + + if ( + rhs.mp_obj_->type == msgpack::type::STR || + rhs.mp_obj_->type == msgpack::type::BIN || + rhs.mp_obj_->type == msgpack::type::ARRAY || + rhs.mp_obj_->type == msgpack::type::MAP || + rhs.mp_obj_->type == msgpack::type::EXT) + { + // The above assignment will only copy the pointer for these + // types. So we need to add the source zones to our own to + // ensure that the data structure pointed to by thet pointer + // continues to exist for as long as we do. + if (mp_zones_ != rhs.mp_zones_) + mp_zones_->insert(mp_zones_->end(), + rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend()); + } } return *this; } From 9bbbe92cc9dd0231b5b9b63f62ce9031256998a0 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 13 Nov 2020 14:16:05 +0100 Subject: [PATCH 06/42] Add Data memory usage regression test --- libmuscle/cpp/build/libmuscle/tests/Makefile | 4 + .../libmuscle/tests/test_data_memory_use.cpp | 98 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp diff --git a/libmuscle/cpp/build/libmuscle/tests/Makefile b/libmuscle/cpp/build/libmuscle/tests/Makefile index e285659a..45304a4e 100644 --- a/libmuscle/cpp/build/libmuscle/tests/Makefile +++ b/libmuscle/cpp/build/libmuscle/tests/Makefile @@ -95,3 +95,7 @@ test_dep_lib_paths := $(subst $(space),:,$(foreach DIR,$(DEP_DIRS),$(DIR)/lib)) run_test%: test% export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; $(VALGRIND) ./$< +# This uses setrlimit, which is not compatible with Valgrind, so run without +.PHONY: run_test_data_memory_use +run_test_data_memory_use: test_data_memory_use + export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; ./$< diff --git a/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp new file mode 100644 index 00000000..e4913fcd --- /dev/null +++ b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp @@ -0,0 +1,98 @@ +#include "libmuscle/data.hpp" +#include "libmuscle/mcp/data_pack.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + + +using libmuscle::impl::Data; + + +int main(int argc, char *argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(libmuscle_mcp_data, large_structure) { + // produce a large with items in it + // regression test for excess memory usage + + // at default malloc settings, we can measure to about 128kB accuracy + const std::size_t granularity = 128u * 1024u; + + // we want to measure to 1% accuracy + const std::size_t data_size = granularity * 100u; + + // we set max overhead at 110%, because msgpack's allocator allocates + // in powers of 2 so we need 100% for that, and a bit of overhead for + // the list itself and overhead. + const std::size_t overhead = data_size * 110 / 100; + + // data segment before allocating is about 200kB + const std::size_t base_size = 200 * 1024; + + const std::size_t max_size = base_size + data_size + overhead; + + // tell the kernel to kill us if we're using too much memory + struct rlimit old_limits, limits; + getrlimit(RLIMIT_DATA, &old_limits); + getrlimit(RLIMIT_DATA, &limits); + limits.rlim_cur = max_size; + setrlimit(RLIMIT_DATA, &limits); + + // This many items are needed to reach the given data size + // A MsgPack object is 24 bytes + const std::size_t num_items = data_size / 24; + + malloc_info(0, stderr); + + Data list = Data::nils(num_items); + malloc_info(0, stderr); + for (std::size_t i = 0; i < num_items; ++i) + list[i] = Data(42424242); + + malloc_info(0, stderr); + + // reset memory limit + setrlimit(RLIMIT_DATA, &old_limits); +} + +TEST(libmuscle_mcp_data, nested_structure) { + // produce a large list with nested structures in it + // regression test for excess memory usage + + // Originally reported with a 10M item list, which would try to use + // 600GB. Scaled down here to speed things up, and we want to do at least + // 100x better than that, so 10k items in 6MB. + + // tell the kernel to kill us if we're using too much memory + struct rlimit old_limits, limits; + getrlimit(RLIMIT_DATA, &old_limits); + getrlimit(RLIMIT_DATA, &limits); + limits.rlim_cur = 6 * 1024 * 1024; + setrlimit(RLIMIT_DATA, &limits); + + const std::size_t num_items = 10000; + malloc_info(0, stderr); + + Data list = Data::nils(num_items); + malloc_info(0, stderr); + for (std::size_t i = 0; i < num_items; ++i) { + auto pos = Data::list(1.2, 3.4, 5.6); + list[i] = Data::list(pos, 7.8, 2); + } + + malloc_info(0, stderr); + + // reset memory limit + setrlimit(RLIMIT_DATA, &old_limits); +} + From 2394ae0566b99dbeb9fb78a1b41e7beea2435da0 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 13 Nov 2020 14:16:28 +0100 Subject: [PATCH 07/42] Fix typos in comments --- libmuscle/cpp/src/libmuscle/data.cpp | 2 +- libmuscle/python/libmuscle/mcp/pipe_multiplexer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp index a69bd2fc..dee33ef6 100644 --- a/libmuscle/cpp/src/libmuscle/data.cpp +++ b/libmuscle/cpp/src/libmuscle/data.cpp @@ -856,7 +856,7 @@ Data & Data::operator=(Data const & rhs) { { // The above assignment will only copy the pointer for these // types. So we need to add the source zones to our own to - // ensure that the data structure pointed to by thet pointer + // ensure that the data structure pointed to by that pointer // continues to exist for as long as we do. if (mp_zones_ != rhs.mp_zones_) mp_zones_->insert(mp_zones_->end(), diff --git a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py index 79e6b94a..2d8e7e85 100644 --- a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py +++ b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py @@ -19,7 +19,7 @@ class _InstancePipe: """Pipes for communicating between an instance and the mux. Objects of this class contain the endpoints for a pipe - that is used to communicate between an instance processe and the + that is used to communicate between an instance process and the multiplexer. The multiplexer (in this module) facilitates the creation of peer-to-peer pipe connections between the processes. From fb8958c400acc8dc3ed9169d2fee10454320ac38 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 13 Nov 2020 15:26:03 +0100 Subject: [PATCH 08/42] Add lists-vs-grids performance note to documentation --- docs/source/cplusplus.rst | 24 ++++++++++++++++++------ docs/source/fortran.rst | 16 ++++++++++++++-- docs/source/tutorial.rst | 12 ++++++++++-- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/source/cplusplus.rst b/docs/source/cplusplus.rst index 6b49f09f..aa83b4a4 100644 --- a/docs/source/cplusplus.rst +++ b/docs/source/cplusplus.rst @@ -334,9 +334,18 @@ objects containing different kinds of data: "existing_object", d4 ); + // grid + // +-----------------+ + // | 1.0 | 2.0 | 3.0 | + // +-----------------+ + // | 4.0 | 5.0 | 6.0 | + // +-----------------+ + std::vector array = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + auto d11 = Data::grid(array.data(), {2, 3}, {"row", "col"}); + // byte array std::vector bytes; - auto d11 = Data::byte_array(bytes.data(), bytes.size()); + auto d12 = Data::byte_array(bytes.data(), bytes.size()); // simple types are created automatically when making a Message instance.send("output_port", Message(t_cur, 1.0)); @@ -344,15 +353,18 @@ objects containing different kinds of data: std::string text("A text message"); instance.send("output_port", Message(t_cur, text)); - // but you have to name dicts, lists, and byte arrays + // but you have to name dicts, lists, grids and byte arrays instance.send("output_port", Message(t_cur, Data::list("a", 10))); As you can see, sending complex data types with MUSCLE is almost as easy in C++ -as it is in Python. One thing that's still missing is sending and receiving -multidimensional arrays of numbers. We hope to add that in a future version; for -now you'll need to put them into a list of numbers manually. - +as it is in Python. There is however a bit of a cost to this ease-of-use in the +form of extra memory usage. If you're sending large amounts of data, then it is +best to use as many grids as you can, as those are much more efficient than +dictionaries and lists. In particular, a set of objects (agents for example) is +much more efficiently sent as a dictionary-of-grids (with one 1D-grid for each +attribute) than as a list of dictionaries. (The latter will work up to a million +objects or so, but it will be slower.) .. code-block:: cpp diff --git a/docs/source/fortran.rst b/docs/source/fortran.rst index cd817464..9ffd4575 100644 --- a/docs/source/fortran.rst +++ b/docs/source/fortran.rst @@ -346,7 +346,8 @@ your model!): .. code-block:: fortran - type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11 + type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12 + integer (LIBMUSCLE_int8), dimension(2, 3) :: ar character(len=1), dimension(1024) :: bytes ! create a Data containing a nil value (None in Python) @@ -377,13 +378,24 @@ your model!): call LIBMUSCLE_Data_set_item(d10, 'note', 'Keys must be strings') call LIBMUSCLE_Data_set_item(d10, 'nest all you want', d9) + ! grid + ar = reshape(spread((/1_LIBMUSCLE_int8/), 1, 6), (/2, 3/)) + d12 = LIBMUSCLE_Data_create_grid(ar) + ! byte array - d11 = LIBMUSCLE_Data_create_byte_array(bytes) + d12 = LIBMUSCLE_Data_create_byte_array(bytes) As you can see, sending complex data types with MUSCLE is a bit more difficult in Fortran than in Python, but it is not too burdensome. +If you want to send large amounts of data, then use grids (arrays) as much as +possible. Lists and dicts are more flexible, but they are also slower and take +up much more memory. In particular, a set of objects (agents for example) is +better sent as a dict of arrays (with one 1D array for each attribute) than as a +list of dicts. (The latter will probably work up to 1 million objects or so, but +it will still be slower.) + Handling errors ``````````````` diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 681189de..1590ba5b 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -248,8 +248,9 @@ and is set to ``None`` here. MUSCLE 3 uses `MessagePack `_ to encode messages between models. MessagePack is a binary encoding format which can be thought of as a binary version of JSON. That means that the message can be an integer, float, -bool, string, or a list or dictionary containing such. Like with JSON, these can -be nested, so you can send a dictionary containing lists of floats for example. +bool, string, or a list or dictionary containing such, and MUSCLE 3 also +supports NumPy arrays and byte arrays. Like with JSON, these can be nested, so +you can send a dictionary containing lists of floats for example. MessagePack is self-describing, so you can inspect the received message to find out what you were sent. In most cases, all data you send on a particular @@ -268,6 +269,13 @@ name of the dimensions. In this case there's only one, and ``x`` is not very descriptive, so we could have also passed ``U`` directly, in which case MUSCLE would have sent a :class:`libmuscle.Grid` without index names automatically. +Note that grids (and NumPy arrays) are much more efficient than lists and +dictionaries. If you have a lot of data to send, then you should use those as +much as possible. For example, a set of agents is best sent as a dictionary of +1D NumPy arrays, with one array/grid for each attribute. A list of dictionaries +will be quite a bit slower and use a lot more memory, but it should work up to +a million or so objects. + Finally, if you want to use your own encoding, you can just send a ``bytes`` object, which will be transmitted as-is, with minimal overhead. From 0d566b67f8e600deacdbb670def38287c6effc14 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 26 Nov 2020 22:03:35 +0100 Subject: [PATCH 09/42] Experiment with BetterCodeHub --- .bettercodehub.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .bettercodehub.yml diff --git a/.bettercodehub.yml b/.bettercodehub.yml new file mode 100644 index 00000000..42d43e86 --- /dev/null +++ b/.bettercodehub.yml @@ -0,0 +1,23 @@ +component_depth: 1 +languages: + - name: python + production: + include: + - /libmuscle/python/libmuscle/.*\.py + - /muscle_manager/.*\.py + exclude: + - .*/test/.*\.py + test: + include: + - .*/test/.*\.py + - /integration_test/.*\.py + - name: cpp + production: + include: + - /libmuscle/cpp/src/libmuscle/.*\.?pp + - /libmuscle/cpp/src/ymmsl/.*\.?pp + exclude: + - .*/tests/.*\.?pp + test: + include: + - .*/tests/.*\.?pp From 64683c91dcc20e1704e0fc038180c4af866f9415 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 26 Nov 2020 22:14:07 +0100 Subject: [PATCH 10/42] Improve BCH configuration --- .bettercodehub.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.bettercodehub.yml b/.bettercodehub.yml index 42d43e86..40631276 100644 --- a/.bettercodehub.yml +++ b/.bettercodehub.yml @@ -2,10 +2,11 @@ component_depth: 1 languages: - name: python production: - include: - - /libmuscle/python/libmuscle/.*\.py - - /muscle_manager/.*\.py exclude: + - /muscle_manager_protocol/.*\.py + - /scripts/.*\.py + - /setup.py + - /docs/.* - .*/test/.*\.py test: include: @@ -13,10 +14,10 @@ languages: - /integration_test/.*\.py - name: cpp production: - include: - - /libmuscle/cpp/src/libmuscle/.*\.?pp - - /libmuscle/cpp/src/ymmsl/.*\.?pp exclude: + - /libmuscle/cpp/src/muscle_manager_protocol/.* + - /libmuscle/cpp/src/libmuscle/bindings/.* + - /libmuscle/cpp/src/ymmsl/bindings/.* - .*/tests/.*\.?pp test: include: From c13c95cd44c3648299aa2348c77056ba5fac99ed Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 26 Nov 2020 22:16:05 +0100 Subject: [PATCH 11/42] Fix Python 3.5 CI issue --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4c771b49..a4bbd767 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,8 @@ 'pytest>=3.5', 'pytest-cov', 'pytest-flake8', - 'pytest-mypy' + 'pytest-mypy', + 'importlib-metadata==2.1.0' ], extras_require={ 'dev': [ From e0cc6ba526da966c70e4b9b75c607e9dc6efa631 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 27 Nov 2020 09:26:28 +0100 Subject: [PATCH 12/42] Remove docs from BCH C++ analysis too --- .bettercodehub.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.bettercodehub.yml b/.bettercodehub.yml index 40631276..cfb0713d 100644 --- a/.bettercodehub.yml +++ b/.bettercodehub.yml @@ -19,6 +19,7 @@ languages: - /libmuscle/cpp/src/libmuscle/bindings/.* - /libmuscle/cpp/src/ymmsl/bindings/.* - .*/tests/.*\.?pp + - /docs/.* test: include: - .*/tests/.*\.?pp From a94ba990432206b97ebcc4a4ecfeb834585a6273 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 21 Dec 2020 09:26:33 +0100 Subject: [PATCH 13/42] Remove CI for Ubuntu 19.10, which cannot be installed any more --- .github/workflows/ci_ubuntu19.10.yaml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .github/workflows/ci_ubuntu19.10.yaml diff --git a/.github/workflows/ci_ubuntu19.10.yaml b/.github/workflows/ci_ubuntu19.10.yaml deleted file mode 100644 index 560d5284..00000000 --- a/.github/workflows/ci_ubuntu19.10.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Run Continuous Integration for the latest Ubuntu release -# This mainly checks for issues/regressions in the native build -name: native_compatibility_ubuntu19.10 -on: - schedule: - - cron: '0 3 * * 0' - push: - branches: - - 'release-*' -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Run tests on Ubuntu 19.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' From 86315817db04d1c71d34a0729066f2f8d3a4da43 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 21 Dec 2020 09:28:30 +0100 Subject: [PATCH 14/42] Update auto-installer to latest OpenSSL --- libmuscle/cpp/build/grpc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmuscle/cpp/build/grpc/Makefile b/libmuscle/cpp/build/grpc/Makefile index 5c688488..ad61c0d1 100644 --- a/libmuscle/cpp/build/grpc/Makefile +++ b/libmuscle/cpp/build/grpc/Makefile @@ -40,7 +40,7 @@ include $(TOOLDIR)/make_available.make dep_name := openssl dep_version_constraint := >= 1.0.2 -dep_version := 1.1.1f +dep_version := 1.1.1i dep_pkgconfig_name := openssl dep_install := 1 include $(TOOLDIR)/make_available.make From 630312d3a3d2312a28cae72ef3d7af1455e55d5f Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 21 Dec 2020 09:43:12 +0100 Subject: [PATCH 15/42] Try to fix Python 3.5 and 3.6 CI --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a4bbd767..fd025417 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ 'grpcio>=1.24.3, <2', 'msgpack', 'netifaces', - 'numpy>=1.12', + 'numpy>=1.12,<1.20', 'protobuf>=3.10.0, <4', 'typing_extensions', 'ymmsl==0.10.1' # Also in CI, update there as well @@ -67,7 +67,7 @@ tests_require=[ 'coverage<5', 'mypy', - 'pytest>=3.5', + 'pytest>=3.5,<6.2', 'pytest-cov', 'pytest-flake8', 'pytest-mypy', From 3f7851f7fbce34626cf505c1942f6622f0a53d68 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 24 Dec 2020 17:24:21 +0100 Subject: [PATCH 16/42] Use development version of ymmsl-python --- .github/workflows/ci.yaml | 2 +- .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.5.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 2 +- .github/workflows/ci_python3.6.yaml | 2 +- .github/workflows/ci_python3.7.yaml | 2 +- .github/workflows/ci_python3.8.yaml | 2 +- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- .github/workflows/ci_ubuntu20.10.yaml | 2 +- integration_test/conftest.py | 9 ++++++--- integration_test/test_all.py | 6 +++--- integration_test/test_cpp_mmp_client.py | 2 +- integration_test/test_duplication_mapper.py | 8 ++++---- integration_test/test_logging.py | 2 +- integration_test/test_parameter_overlays.py | 12 ++++++------ libmuscle/python/libmuscle/communicator.py | 12 ++++++------ libmuscle/python/libmuscle/manager/manager.py | 2 +- .../python/libmuscle/manager/test/conftest.py | 12 ++++++------ .../python/libmuscle/manager/topology_store.py | 14 +++++++------- libmuscle/python/libmuscle/peer_manager.py | 4 ++-- libmuscle/python/libmuscle/runner.py | 2 +- setup.py | 2 +- 24 files changed, 56 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f2527ba..c9d9c8ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev - pip install ymmsl==0.10.1 + pip install git+https://github.com/multiscale/ymmsl-python.git@develop - name: Build and run the test suite env: diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 414110fb..7732ce93 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 3d79cf5f..6545318b 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index 75fe0a25..71125125 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6.1-eggs - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml index eab001e9..5ea1eb2d 100644 --- a/.github/workflows/ci_python3.6.yaml +++ b/.github/workflows/ci_python3.6.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6-eggs - name: Run Python tests on 3.6 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml index 88fbd066..b4698a9f 100644 --- a/.github/workflows/ci_python3.7.yaml +++ b/.github/workflows/ci_python3.7.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.7-eggs - name: Run Python tests on 3.7 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml index 549b2d38..841cfa62 100644 --- a/.github/workflows/ci_python3.8.yaml +++ b/.github/workflows/ci_python3.8.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.8-eggs - name: Run Python tests on 3.8 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 7d783789..7dcda949 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 3d9b313f..9ec37144 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index ea711557..560ad5d9 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml index 285eb940..f562dd20 100644 --- a/.github/workflows/ci_ubuntu20.10.yaml +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/integration_test/conftest.py b/integration_test/conftest.py index a64b0717..b2ae93f3 100644 --- a/integration_test/conftest.py +++ b/integration_test/conftest.py @@ -67,7 +67,7 @@ def mmp_server_process(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' @@ -84,6 +84,9 @@ def mmp_server_process(yatiml_log_warning): ' test6:\n' ' - [1.0, 2.0]\n' ' - [3.0, 1.0]\n' + 'implementations:\n' + ' macro_implementation: macro.py\n' + ' micro_implementation: micro.py\n' ) ymmsl_doc = ymmsl.load(ymmsl_text) @@ -96,7 +99,7 @@ def mmp_server_process_simple(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro: micro_implementation\n' ' conduits:\n' @@ -123,7 +126,7 @@ def mmp_server(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' diff --git a/integration_test/test_all.py b/integration_test/test_all.py index f314396a..8764f3a4 100644 --- a/integration_test/test_all.py +++ b/integration_test/test_all.py @@ -1,7 +1,7 @@ from collections import OrderedDict import numpy as np -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, +from ymmsl import (Component, Conduit, Configuration, Model, Operator, Settings) from libmuscle import Grid, Instance, Message @@ -63,8 +63,8 @@ def test_all(log_file_in_tmpdir): """A positive all-up test of everything. """ elements = [ - ComputeElement('macro', 'macro_impl'), - ComputeElement('micro', 'micro_impl', [10])] + Component('macro', 'macro_impl'), + Component('micro', 'micro_impl', [10])] conduits = [ Conduit('macro.out', 'micro.in'), diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py index 1c51de0a..1f208741 100644 --- a/integration_test/test_cpp_mmp_client.py +++ b/integration_test/test_cpp_mmp_client.py @@ -20,7 +20,7 @@ def do_mmp_client_test(caplog): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' diff --git a/integration_test/test_duplication_mapper.py b/integration_test/test_duplication_mapper.py index 36dbf3ba..9717ad36 100644 --- a/integration_test/test_duplication_mapper.py +++ b/integration_test/test_duplication_mapper.py @@ -1,4 +1,4 @@ -from ymmsl import (ComputeElement, Conduit, Configuration, Operator, Model, +from ymmsl import (Component, Conduit, Configuration, Operator, Model, Settings) from libmuscle import Instance, Message @@ -36,9 +36,9 @@ def test_duplication_mapper(log_file_in_tmpdir): This is an acyclic workflow. """ elements = [ - ComputeElement('dm', 'muscle.duplication_mapper'), - ComputeElement('first', 'receiver'), - ComputeElement('second', 'receiver')] + Component('dm', 'muscle.duplication_mapper'), + Component('first', 'receiver'), + Component('second', 'receiver')] conduits = [ Conduit('dm.out', 'first.in'), diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index 1c92a694..73b7b581 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -16,7 +16,7 @@ def do_logging_test(caplog): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' diff --git a/integration_test/test_parameter_overlays.py b/integration_test/test_parameter_overlays.py index a4211cef..cf091193 100644 --- a/integration_test/test_parameter_overlays.py +++ b/integration_test/test_parameter_overlays.py @@ -1,6 +1,6 @@ from collections import OrderedDict -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, +from ymmsl import (Component, Conduit, Configuration, Model, Operator, Settings) from libmuscle import Instance, Message @@ -94,11 +94,11 @@ def test_settings_overlays(log_file_in_tmpdir): """A positive all-up test of settings overlays. """ elements = [ - ComputeElement('qmc', 'qmc'), - ComputeElement('macro', 'macro', [10]), - ComputeElement('relay', 'explicit_relay'), - ComputeElement('relay2', 'explicit_relay'), - ComputeElement('micro', 'micro', [10])] + Component('qmc', 'qmc'), + Component('macro', 'macro', [10]), + Component('relay', 'explicit_relay'), + Component('relay2', 'explicit_relay'), + Component('micro', 'micro', [10])] conduits = [ Conduit('qmc.settings_out', 'macro.muscle_settings_in'), diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py index 49f3ec06..0391501f 100644 --- a/libmuscle/python/libmuscle/communicator.py +++ b/libmuscle/python/libmuscle/communicator.py @@ -396,16 +396,16 @@ def __ports_from_conduits(self, conduits: List[Conduit] """ ports = dict() for conduit in conduits: - if conduit.sending_compute_element() == self._kernel: + if conduit.sending_component() == self._kernel: port_id = conduit.sending_port() operator = Operator.O_F port_peer_dims = self._peer_manager.get_peer_dims( - conduit.receiving_compute_element()) - elif conduit.receiving_compute_element() == self._kernel: + conduit.receiving_component()) + elif conduit.receiving_component() == self._kernel: port_id = conduit.receiving_port() operator = Operator.F_INIT port_peer_dims = self._peer_manager.get_peer_dims( - conduit.sending_compute_element()) + conduit.sending_component()) else: continue @@ -425,14 +425,14 @@ def __settings_in_port(self, conduits: List[Conduit]) -> Port: conduits: The list of conduits. """ for conduit in conduits: - if conduit.receiving_compute_element() == self._kernel: + if conduit.receiving_component() == self._kernel: port_id = conduit.receiving_port() if str(port_id) == 'muscle_settings_in': return Port(str(port_id), Operator.F_INIT, False, self._peer_manager.is_connected(port_id), len(self._index), self._peer_manager.get_peer_dims( - conduit.sending_compute_element())) + conduit.sending_component())) return Port('muscle_settings_in', Operator.F_INIT, False, False, len(self._index), []) diff --git a/libmuscle/python/libmuscle/manager/manager.py b/libmuscle/python/libmuscle/manager/manager.py index 96116a02..2b3637e6 100644 --- a/libmuscle/python/libmuscle/manager/manager.py +++ b/libmuscle/python/libmuscle/manager/manager.py @@ -45,7 +45,7 @@ def generate_indices(multiplicity: List[int]) -> List[str]: return indices result = list() # type: List[str] - for element in model.compute_elements: + for element in model.components: if len(element.multiplicity) == 0: result.append(str(element.name)) else: diff --git a/libmuscle/python/libmuscle/manager/test/conftest.py b/libmuscle/python/libmuscle/manager/test/conftest.py index 8d05a800..fa981af1 100644 --- a/libmuscle/python/libmuscle/manager/test/conftest.py +++ b/libmuscle/python/libmuscle/manager/test/conftest.py @@ -1,5 +1,5 @@ import pytest -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Reference, +from ymmsl import (Component, Conduit, Configuration, Model, Reference, Settings) from libmuscle.manager.instance_registry import InstanceRegistry @@ -34,8 +34,8 @@ def topology_store() -> TopologyStore: Model( 'test_model', [ - ComputeElement('macro', 'macro_implementation'), - ComputeElement( + Component('macro', 'macro_implementation'), + Component( 'micro', 'micro_implementation', [10, 10])], [ Conduit('macro.out', 'micro.in'), @@ -75,9 +75,9 @@ def topology_store2() -> TopologyStore: Model( 'test_model', [ - ComputeElement('macro', 'macro_implementation'), - ComputeElement('meso', 'meso_implementation', [5]), - ComputeElement('micro', 'micro_implementation', [5, 10]) + Component('macro', 'macro_implementation'), + Component('meso', 'meso_implementation', [5]), + Component('micro', 'micro_implementation', [5, 10]) ], [ Conduit('macro.out', 'meso.in'), diff --git a/libmuscle/python/libmuscle/manager/topology_store.py b/libmuscle/python/libmuscle/manager/topology_store.py index d3750f01..874394f2 100644 --- a/libmuscle/python/libmuscle/manager/topology_store.py +++ b/libmuscle/python/libmuscle/manager/topology_store.py @@ -28,7 +28,7 @@ def __init__(self, config: Configuration) -> None: self.conduits = config.model.conduits self.kernel_dimensions = { k.name: k.multiplicity - for k in config.model.compute_elements} + for k in config.model.components} def has_kernel(self, kernel: Reference) -> bool: """Returns True iff the given kernel is in the model. @@ -49,9 +49,9 @@ def get_conduits(self, kernel_name: Reference) -> List[Conduit]: """ ret = list() for conduit in self.conduits: - if conduit.sending_compute_element() == kernel_name: + if conduit.sending_component() == kernel_name: ret.append(conduit) - if conduit.receiving_compute_element() == kernel_name: + if conduit.receiving_component() == kernel_name: ret.append(conduit) return ret @@ -70,10 +70,10 @@ def get_peer_dimensions(self, kernel_name: Reference """ ret = dict() for conduit in self.conduits: - if conduit.sending_compute_element() == kernel_name: - recv = conduit.receiving_compute_element() + if conduit.sending_component() == kernel_name: + recv = conduit.receiving_component() ret[recv] = self.kernel_dimensions[recv] - if conduit.receiving_compute_element() == kernel_name: - snd = conduit.sending_compute_element() + if conduit.receiving_component() == kernel_name: + snd = conduit.sending_component() ret[snd] = self.kernel_dimensions[snd] return ret diff --git a/libmuscle/python/libmuscle/peer_manager.py b/libmuscle/python/libmuscle/peer_manager.py index 7066f244..57c50121 100644 --- a/libmuscle/python/libmuscle/peer_manager.py +++ b/libmuscle/python/libmuscle/peer_manager.py @@ -37,10 +37,10 @@ def __init__(self, kernel: Reference, index: List[int], self.__peers = dict() # type: Dict[Reference, Reference] for conduit in conduits: - if str(conduit.sending_compute_element()) == str(kernel): + if str(conduit.sending_component()) == str(kernel): # we send on the port this conduit attaches to self.__peers[conduit.sender] = conduit.receiver - if str(conduit.receiving_compute_element()) == str(kernel): + if str(conduit.receiving_component()) == str(kernel): # we receive on the port this conduit attaches to self.__peers[conduit.receiver] = conduit.sender diff --git a/libmuscle/python/libmuscle/runner.py b/libmuscle/python/libmuscle/runner.py index 3e422262..2e725c3d 100644 --- a/libmuscle/python/libmuscle/runner.py +++ b/libmuscle/python/libmuscle/runner.py @@ -246,7 +246,7 @@ def run_simulation( ' definition, so the simulation can not be run.') instances = dict() - for ce in configuration.model.compute_elements: + for ce in configuration.model.components: impl_name = str(ce.implementation) if impl_name not in implementations: raise ValueError(('The model specifies an implementation named' diff --git a/setup.py b/setup.py index fd025417..102ddf52 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ 'numpy>=1.12,<1.20', 'protobuf>=3.10.0, <4', 'typing_extensions', - 'ymmsl==0.10.1' # Also in CI, update there as well + 'ymmsl' # Also in CI, update there as well ], setup_requires=[ 'pytest-runner', From f91b0cb6d7f9863836e3b935d2b326437273c8e2 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 24 Dec 2020 17:56:14 +0100 Subject: [PATCH 17/42] Try fixing Actions by clearing cache --- .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 7732ce93..82a45a81 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}/.eggs - key: python-compatibility-3.5.1-eggs + key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index 71125125..a4106e25 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}/.eggs - key: python-compatibility-3.6.1-eggs + key: python-compatibility-3.6.1-eggs2 - name: Run Python tests on 3.6.1 latest run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' From 1cdad40e1c40aa89e86f860e89d73773351ae484 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 24 Dec 2020 18:03:16 +0100 Subject: [PATCH 18/42] Try to fix CI on Python 3.5.1 and 3.6.1 --- .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 82a45a81..5b6acb43 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index a4106e25..508fbb05 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6.1-eggs2 - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' From 78f769736b6c4ecbbb5f5356569a6b5415aeda0a Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 24 Dec 2020 20:08:47 +0100 Subject: [PATCH 19/42] Update examples to new ymmsl format --- docs/source/examples/reaction_diffusion.ymmsl | 2 +- docs/source/examples/reaction_diffusion_mc.ymmsl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/examples/reaction_diffusion.ymmsl b/docs/source/examples/reaction_diffusion.ymmsl index 5cf3468b..898924b8 100644 --- a/docs/source/examples/reaction_diffusion.ymmsl +++ b/docs/source/examples/reaction_diffusion.ymmsl @@ -2,7 +2,7 @@ ymmsl_version: v0.1 model: name: reaction_diffusion - compute_elements: + components: macro: diffusion micro: reaction conduits: diff --git a/docs/source/examples/reaction_diffusion_mc.ymmsl b/docs/source/examples/reaction_diffusion_mc.ymmsl index 4e50db94..ccfa2f6b 100644 --- a/docs/source/examples/reaction_diffusion_mc.ymmsl +++ b/docs/source/examples/reaction_diffusion_mc.ymmsl @@ -2,7 +2,7 @@ ymmsl_version: v0.1 model: name: reaction_diffusion_mc - compute_elements: + components: mc: mc_driver rr: load_balancer macro: From 7f20a15131558df5059279b2f7c7ce3b95335345 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 24 Dec 2020 20:10:29 +0100 Subject: [PATCH 20/42] Accept multiple yMMSL files on the muscle_manager command line --- docs/source/examples/rd_settings.ymmsl | 14 ++++++++++++++ docs/source/examples/reaction_diffusion.ymmsl | 10 ---------- docs/source/examples/reaction_diffusion_cpp.sh | 2 +- .../examples/reaction_diffusion_fortran.sh | 2 +- .../examples/reaction_diffusion_mpi_cpp.sh | 2 +- .../examples/reaction_diffusion_mpi_fortran.sh | 2 +- .../source/examples/reaction_diffusion_python.sh | 2 +- .../examples/reaction_diffusion_python_cpp.sh | 2 +- .../reaction_diffusion_python_fortran.sh | 2 +- muscle_manager/muscle_manager.py | 16 ++++++++++++---- setup.py | 2 +- 11 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 docs/source/examples/rd_settings.ymmsl diff --git a/docs/source/examples/rd_settings.ymmsl b/docs/source/examples/rd_settings.ymmsl new file mode 100644 index 00000000..3aefddde --- /dev/null +++ b/docs/source/examples/rd_settings.ymmsl @@ -0,0 +1,14 @@ +ymmsl_version: v0.1 + +model: + name: reaction_diffusion + +settings: + micro.t_max: 2.469136e-06 + micro.dt: 2.469136e-08 + macro.t_max: 0.0001234568 + macro.dt: 2.469136e-06 + x_max: 1.0 + dx: 0.01 + k: -40500.0 + d: 0.0405 diff --git a/docs/source/examples/reaction_diffusion.ymmsl b/docs/source/examples/reaction_diffusion.ymmsl index 898924b8..c75ea225 100644 --- a/docs/source/examples/reaction_diffusion.ymmsl +++ b/docs/source/examples/reaction_diffusion.ymmsl @@ -8,13 +8,3 @@ model: conduits: macro.state_out: micro.initial_state micro.final_state: macro.state_in - -settings: - micro.t_max: 2.469136e-06 - micro.dt: 2.469136e-08 - macro.t_max: 0.0001234568 - macro.dt: 2.469136e-06 - x_max: 1.0 - dx: 0.01 - k: -40500.0 - d: 0.0405 diff --git a/docs/source/examples/reaction_diffusion_cpp.sh b/docs/source/examples/reaction_diffusion_cpp.sh index 4e421527..f984a421 100755 --- a/docs/source/examples/reaction_diffusion_cpp.sh +++ b/docs/source/examples/reaction_diffusion_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_fortran.sh b/docs/source/examples/reaction_diffusion_fortran.sh index bdce14eb..6792da11 100755 --- a/docs/source/examples/reaction_diffusion_fortran.sh +++ b/docs/source/examples/reaction_diffusion_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_mpi_cpp.sh b/docs/source/examples/reaction_diffusion_mpi_cpp.sh index dcc18da1..338ae0b8 100755 --- a/docs/source/examples/reaction_diffusion_mpi_cpp.sh +++ b/docs/source/examples/reaction_diffusion_mpi_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion with MPI in C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_mpi_fortran.sh b/docs/source/examples/reaction_diffusion_mpi_fortran.sh index acdd6566..394b74d4 100755 --- a/docs/source/examples/reaction_diffusion_mpi_fortran.sh +++ b/docs/source/examples/reaction_diffusion_mpi_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion MPI in Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python.sh b/docs/source/examples/reaction_diffusion_python.sh index 843dbf11..7f9ff37d 100755 --- a/docs/source/examples/reaction_diffusion_python.sh +++ b/docs/source/examples/reaction_diffusion_python.sh @@ -1,7 +1,7 @@ #!/bin/bash . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python_cpp.sh b/docs/source/examples/reaction_diffusion_python_cpp.sh index f04c0276..bbff9fab 100755 --- a/docs/source/examples/reaction_diffusion_python_cpp.sh +++ b/docs/source/examples/reaction_diffusion_python_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Python and C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python_fortran.sh b/docs/source/examples/reaction_diffusion_python_fortran.sh index 5aef427a..60b41469 100755 --- a/docs/source/examples/reaction_diffusion_python_fortran.sh +++ b/docs/source/examples/reaction_diffusion_python_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Python and Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py index 6385687b..8f84ad98 100644 --- a/muscle_manager/muscle_manager.py +++ b/muscle_manager/muscle_manager.py @@ -1,14 +1,22 @@ +from typing import Sequence + import click import ymmsl +from ymmsl import Configuration from libmuscle.manager.manager import start_server @click.command() -@click.argument('ymmsl_file') -def manage_simulation(ymmsl_file: str) -> None: - with open(ymmsl_file) as f: - configuration = ymmsl.load(f) +@click.argument( + 'ymmsl_files', nargs=-1, required=True, type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, + allow_dash=True, resolve_path=True)) +def manage_simulation(ymmsl_files: Sequence[str]) -> None: + configuration = Configuration() + for path in ymmsl_files: + with open(path, 'r') as f: + configuration.update(ymmsl.load(f)) server = start_server(configuration) print(server.get_location()) diff --git a/setup.py b/setup.py index 102ddf52..9ae50380 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ }, python_requires='>=3.5, <4', install_requires=[ - 'click', + 'click>=6', 'grpcio>=1.24.3, <2', 'msgpack', 'netifaces', From 0427875ed0709c44451511bbed8088bd7533dcf6 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 28 Dec 2020 10:14:25 +0100 Subject: [PATCH 21/42] Try to fix compatibility CI --- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- .github/workflows/ci_ubuntu20.10.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 7dcda949..5b1cb636 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 9ec37144..7031b82b 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index 560ad5d9..fc5b3a57 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml index f562dd20..d2ce7f89 100644 --- a/.github/workflows/ci_ubuntu20.10.yaml +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' From f92f53106436174412ecae9a54f11bbf928fb95c Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 5 Jan 2021 20:08:11 +0100 Subject: [PATCH 22/42] Work around mypy issue --- libmuscle/python/libmuscle/instance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index 13338b2d..fbba1828 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -20,6 +20,9 @@ _logger = logging.getLogger(__name__) +_FInitCacheType = Dict[Tuple[str, Optional[int]], Message] + + class Instance: """Represents a compute element instance in a MUSCLE3 simulation. @@ -62,8 +65,7 @@ def __init__(self, ports: Optional[Dict[Operator, List[str]]] = None self._first_run = True """Keeps track of whether this is the first reuse run.""" - FInitCacheType = Dict[Tuple[str, Optional[int]], Message] - self._f_init_cache = dict() # type: FInitCacheType + self._f_init_cache = dict() # type: _FInitCacheType self._register() self._connect() From e571e586aa42402e939113a59ab772dd0910226c Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 5 Jan 2021 20:22:39 +0100 Subject: [PATCH 23/42] Try to fix CI by having it use YAtiML @develop --- .github/workflows/ci.yaml | 1 + .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.5.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 2 +- .github/workflows/ci_python3.6.yaml | 2 +- .github/workflows/ci_python3.7.yaml | 2 +- .github/workflows/ci_python3.8.yaml | 2 +- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c9d9c8ba..93e88b85 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,6 +18,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev + pip install git+https://github.com/yatiml/yatiml.git@develop pip install git+https://github.com/multiscale/ymmsl-python.git@develop - name: Build and run the test suite diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 5b6acb43..b514b9f6 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 6545318b..1b16a043 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" pip install -U git+https://github.com/yatiml/yatiml.git@develop && git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index 508fbb05..d3be4c23 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6.1-eggs2 - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml index 5ea1eb2d..78605b61 100644 --- a/.github/workflows/ci_python3.6.yaml +++ b/.github/workflows/ci_python3.6.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6-eggs - name: Run Python tests on 3.6 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml index b4698a9f..d48d7bb3 100644 --- a/.github/workflows/ci_python3.7.yaml +++ b/.github/workflows/ci_python3.7.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.7-eggs - name: Run Python tests on 3.7 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml index 841cfa62..9608db8b 100644 --- a/.github/workflows/ci_python3.8.yaml +++ b/.github/workflows/ci_python3.8.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.8-eggs - name: Run Python tests on 3.8 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 5b1cb636..b757d1c7 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 7031b82b..7502bfe9 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index fc5b3a57..f150e21a 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' From 47dfd06f50540536aeb101313ce189780eee32a1 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 5 Jan 2021 21:56:25 +0100 Subject: [PATCH 24/42] Try to fix CI mypy run --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4742cd5b..0def4984 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ flake8-ignore = libmuscle/manager_protocol/*.py ALL [mypy] -paths = muscle_manager libmuscle/python mypy_path = libmuscle/python:muscle_manager_protocol warn_unused_configs = True disallow_subclassing_any = True From 8e1e8f0b3e36eb88a24302fbdb3d966958f802db Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 6 Jan 2021 11:03:54 +0100 Subject: [PATCH 25/42] Try to get a bit more error output on the CI --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0def4984..5c62d746 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ test = pytest [tool:pytest] testpaths = muscle_manager libmuscle/python integration_test -addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing -x +addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing flake8-ignore = setup.py E501 muscle_manager/protocol/*.py ALL From 8fe0986c8161e24c3eec0a13cc3b2be4ea5af654 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 6 Jan 2021 11:16:55 +0100 Subject: [PATCH 26/42] Use new PartialConfiguration class to merge ymmsl files --- muscle_manager/muscle_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py index 8f84ad98..a935186a 100644 --- a/muscle_manager/muscle_manager.py +++ b/muscle_manager/muscle_manager.py @@ -2,7 +2,7 @@ import click import ymmsl -from ymmsl import Configuration +from ymmsl import PartialConfiguration from libmuscle.manager.manager import start_server @@ -13,12 +13,12 @@ exists=True, file_okay=True, dir_okay=False, readable=True, allow_dash=True, resolve_path=True)) def manage_simulation(ymmsl_files: Sequence[str]) -> None: - configuration = Configuration() + configuration = PartialConfiguration() for path in ymmsl_files: with open(path, 'r') as f: configuration.update(ymmsl.load(f)) - server = start_server(configuration) + server = start_server(configuration.as_configuration()) print(server.get_location()) server.wait() From ca80d7a36913629837b9668858c1cedf5a2a6e4b Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 6 Jan 2021 11:20:33 +0100 Subject: [PATCH 27/42] Fix typo in CI python 3.5 config --- .github/workflows/ci_python3.5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 1b16a043..3e1177b0 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" pip install -U git+https://github.com/yatiml/yatiml.git@develop && git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' From 25d7b3cd81a99e6f50fece1b8366e1f24b202d80 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 10:28:29 +0100 Subject: [PATCH 28/42] Use daemon threads in Python to avoid hanging when we crash --- libmuscle/python/libmuscle/mcp/pipe_server.py | 4 ++-- libmuscle/python/libmuscle/mcp/tcp_server.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libmuscle/python/libmuscle/mcp/pipe_server.py b/libmuscle/python/libmuscle/mcp/pipe_server.py index 3c22eb42..d0a57da2 100644 --- a/libmuscle/python/libmuscle/mcp/pipe_server.py +++ b/libmuscle/python/libmuscle/mcp/pipe_server.py @@ -31,7 +31,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice self._shutdown_conn, self._handler_shutdown_conn = mp.Pipe() self._server_thread = threading.Thread( target=self.__conn_request_handler, - name='PipeServer-{}'.format(instance_id)) + name='PipeServer-{}'.format(instance_id), daemon=True) self._server_thread.start() else: @@ -72,7 +72,7 @@ def __conn_request_handler(self) -> None: target=self.__mcp_pipe_handler, args=(client_id, connection), name='PipeHandler-{}-{}'.format( - self._instance_id, client_id)) + self._instance_id, client_id), daemon=True) conn_thread.start() conn_threads.append(conn_thread) except EOFError: diff --git a/libmuscle/python/libmuscle/mcp/tcp_server.py b/libmuscle/python/libmuscle/mcp/tcp_server.py index 5385ce48..ea312101 100644 --- a/libmuscle/python/libmuscle/mcp/tcp_server.py +++ b/libmuscle/python/libmuscle/mcp/tcp_server.py @@ -13,6 +13,8 @@ class TcpServerImpl(ss.ThreadingMixIn, ss.TCPServer): + daemon_threads = True + def __init__(self, host_port_tuple: Tuple[str, int], streamhandler: Type, tcp_server: 'TcpServer' ) -> None: @@ -65,7 +67,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice self._server = TcpServerImpl(('', 0), TcpHandler, self) self._server_thread = threading.Thread( - target=self._server.serve_forever) + target=self._server.serve_forever, daemon=True) self._server_thread.start() def get_location(self) -> str: From 94a5d92141dd83d111c987589df72d92074b106d Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 10:53:30 +0100 Subject: [PATCH 29/42] Make logging tests a bit more flexible --- integration_test/test_cpp_mmp_client.py | 10 ++++++---- integration_test/test_logging.py | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py index 1f208741..54faaafe 100644 --- a/integration_test/test_cpp_mmp_client.py +++ b/integration_test/test_cpp_mmp_client.py @@ -83,10 +83,12 @@ def mock_remove(name: Reference): assert result.returncode == 0 # check submit_log_message - assert caplog.records[0].name == 'instances.test_logging' - assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' - assert caplog.records[0].levelname == 'CRITICAL' - assert caplog.records[0].message == 'Integration testing' + for rec in caplog.records: + if rec.name == 'instances.test_logging': + assert rec.time_stamp == '1970-01-01T00:00:02Z' + assert rec.levelname == 'CRITICAL' + assert rec.message == 'Integration testing' + break # check register_instance assert (instance_registry.get_locations('micro[3]') == diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index 73b7b581..c5ced013 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -54,10 +54,12 @@ def do_logging_test(caplog): # log and check client.submit_log_message(message) - assert caplog.records[0].name == 'instances.test_logging' - assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' - assert caplog.records[0].levelname == 'DEBUG' - assert caplog.records[0].message == 'Integration testing' + for rec in caplog.records: + if rec.name == 'instances.test_logging': + assert rec.time_stamp == '1970-01-01T00:00:02Z' + assert rec.levelname == 'DEBUG' + assert rec.message == 'Integration testing' + break server.stop() From 5c5967031ebad92c1ba0589050d74a74731aa961 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 11:14:52 +0100 Subject: [PATCH 30/42] Remove private private variables in MMPServer --- integration_test/test_registration.py | 2 +- .../python/libmuscle/manager/mmp_server.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration_test/test_registration.py b/integration_test/test_registration.py index 77cc81e9..3ed68d96 100644 --- a/integration_test/test_registration.py +++ b/integration_test/test_registration.py @@ -14,7 +14,7 @@ def test_registration(log_file_in_tmpdir, mmp_server): client.register_instance(instance_name, ['tcp://localhost:10000'], [port]) - servicer = mmp_server._MMPServer__servicer + servicer = mmp_server._servicer registry = servicer._MMPServicer__instance_registry assert registry.get_locations(instance_name) == ['tcp://localhost:10000'] diff --git a/libmuscle/python/libmuscle/manager/mmp_server.py b/libmuscle/python/libmuscle/manager/mmp_server.py index 14e9775f..173f1ff8 100644 --- a/libmuscle/python/libmuscle/manager/mmp_server.py +++ b/libmuscle/python/libmuscle/manager/mmp_server.py @@ -252,14 +252,14 @@ def __init__( instance_registry: InstanceRegistry, topology_store: TopologyStore ) -> None: - self.__instance_registry = instance_registry - self.__servicer = MMPServicer(logger, settings, instance_registry, - topology_store) - self.__server = grpc.server(futures.ThreadPoolExecutor()) + self._instance_registry = instance_registry + self._servicer = MMPServicer(logger, settings, instance_registry, + topology_store) + self._server = grpc.server(futures.ThreadPoolExecutor()) mmp_grpc.add_MuscleManagerServicer_to_server( # type: ignore - self.__servicer, self.__server) - self.__server.add_insecure_port('[::]:9000') - self.__server.start() + self._servicer, self._server) + self._server.add_insecure_port('[::]:9000') + self._server.start() def get_location(self) -> str: """Returns this server's network location. @@ -275,9 +275,9 @@ def wait(self) -> None: The server will shut down after every instance has been registered and deregistered again. """ - self.__instance_registry.wait() + self._instance_registry.wait() time.sleep(1) - self.__server.stop(5) + self._server.stop(5) def stop(self) -> None: """Stops the server. @@ -285,4 +285,4 @@ def stop(self) -> None: This makes the server stop serving requests, and shuts down its background threads. """ - self.__server.stop(0) + self._server.stop(0) From 44e97701c87265126c4c148450a95af17694c356 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 11:26:11 +0100 Subject: [PATCH 31/42] Add MUSCLE_INSTANCE environment variable support --- libmuscle/python/libmuscle/instance.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index fbba1828..0c4e933b 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -1,5 +1,6 @@ from copy import copy import logging +import os import sys from typing import cast, Dict, List, Optional, Tuple @@ -509,9 +510,14 @@ def split_reference(ref: Reference) -> Tuple[Reference, List[int]]: name, index = split_reference(prefix_ref) break else: - raise RuntimeError(('A --muscle-instance command line argument is' - ' required to identify this instance. Please' - ' add one.')) + if 'MUSCLE_INSTANCE' in os.environ: + prefix_ref = Reference(os.environ['MUSCLE_INSTANCE']) + name, index = split_reference(prefix_ref) + else: + raise RuntimeError(( + 'A --muscle-instance command line argument or' + ' MUSCLE_INSTANCE environment variable is required to' + ' identify this instance. Please add one.')) return name, index def __list_declared_ports(self) -> List[Port]: From 91a5dd8c7630f80da68ddf7d11a156d7ed96de18 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 12:25:27 +0100 Subject: [PATCH 32/42] Add dir argument to extract_log_file_location --- libmuscle/python/libmuscle/instance.py | 4 +++- libmuscle/python/libmuscle/manager/logger.py | 3 ++- libmuscle/python/libmuscle/util.py | 9 +++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index 0c4e933b..23923969 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -1,6 +1,7 @@ from copy import copy import logging import os +from pathlib import Path import sys from typing import cast, Dict, List, Optional, Tuple @@ -409,7 +410,8 @@ def __set_up_logging(self) -> None: """ id_str = str(self._instance_name()) - logfile = extract_log_file_location('muscle3.{}.log'.format(id_str)) + logfile = extract_log_file_location( + Path.cwd(), 'muscle3.{}.log'.format(id_str)) local_handler = logging.FileHandler(str(logfile), mode='w') formatter = logging.Formatter('%(asctime)-15s: %(name)s' ' %(levelname)s: %(message)s') diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index b91f321f..09e66612 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from libmuscle.logging import LogLevel, Timestamp from libmuscle.util import extract_log_file_location @@ -11,7 +12,7 @@ class Logger: standard out. """ def __init__(self) -> None: - logfile = extract_log_file_location('muscle3_manager.log') + logfile = extract_log_file_location(Path.cwd(), 'muscle3_manager.log') self._local_handler = logging.FileHandler(str(logfile), mode='w') formatter = logging.Formatter('%(time_stamp)-15s: %(name)s' ' %(levelname)s: %(message)s') diff --git a/libmuscle/python/libmuscle/util.py b/libmuscle/python/libmuscle/util.py index bcf27239..5afe8c35 100644 --- a/libmuscle/python/libmuscle/util.py +++ b/libmuscle/python/libmuscle/util.py @@ -1,6 +1,6 @@ from pathlib import Path import sys -from typing import Generator, List, Optional, cast +from typing import Generator, List, cast from ymmsl import Conduit, Reference @@ -90,7 +90,7 @@ def increment_index(index: List[int], dims: List[int]) -> bool: return False -def extract_log_file_location(filename: str) -> Optional[Path]: +def extract_log_file_location(run_dir: Path, filename: str) -> Path: """Gets the log file location from the command line. Extracts the --muscle-log-file= argument to tell the @@ -100,10 +100,11 @@ def extract_log_file_location(filename: str) -> Optional[Path]: directory, will be written inside of that directory, if the path is not an existing directory, then it will be used as the name of the log file to write to. If no command line - argument is given, will be written in the current + argument is given, will be written in the specified directory. Args: + run_dir: Default directory to use. filename: Default file name to use. Returns: @@ -119,7 +120,7 @@ def extract_log_file_location(filename: str) -> Optional[Path]: given_path_str = arg[len(prefix):] if given_path_str == '': - return Path('.') / filename + return run_dir / filename given_path = Path(given_path_str) From ccd416a302433802af38c92224e8a04cf975ee0a Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 12:31:12 +0100 Subject: [PATCH 33/42] Accept a PartialConfiguration in TopologyStore --- libmuscle/python/libmuscle/manager/topology_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmuscle/python/libmuscle/manager/topology_store.py b/libmuscle/python/libmuscle/manager/topology_store.py index 874394f2..c8e3f864 100644 --- a/libmuscle/python/libmuscle/manager/topology_store.py +++ b/libmuscle/python/libmuscle/manager/topology_store.py @@ -1,6 +1,6 @@ from typing import Dict, List -from ymmsl import Conduit, Configuration, Model, Reference +from ymmsl import Conduit, PartialConfiguration, Model, Reference class TopologyStore: @@ -12,7 +12,7 @@ class TopologyStore: Attributes: conduits (List[Conduit]): A list of conduits. """ - def __init__(self, config: Configuration) -> None: + def __init__(self, config: PartialConfiguration) -> None: """Creates a TopologyStore. Creates a TopologyStore containing conduits read from the given From 461175ca0917aaee1142ebc46d2782e3b3f32396 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 12:47:27 +0100 Subject: [PATCH 34/42] Add log_dir argument to manager.Logger() --- libmuscle/python/libmuscle/manager/logger.py | 10 ++++++++-- libmuscle/python/libmuscle/manager/test/conftest.py | 6 ++++-- libmuscle/python/libmuscle/manager/test/test_logger.py | 5 +++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index 09e66612..2f9fa817 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import Optional from libmuscle.logging import LogLevel, Timestamp from libmuscle.util import extract_log_file_location @@ -10,9 +11,14 @@ class Logger: The Logger component takes log messages and writes them to standard out. + + Args: + log_dir: Directory to write the log file into. """ - def __init__(self) -> None: - logfile = extract_log_file_location(Path.cwd(), 'muscle3_manager.log') + def __init__(self, log_dir: Optional[Path] = None) -> None: + if log_dir is None: + log_dir = Path.cwd() + logfile = extract_log_file_location(log_dir, 'muscle3_manager.log') self._local_handler = logging.FileHandler(str(logfile), mode='w') formatter = logging.Formatter('%(time_stamp)-15s: %(name)s' ' %(levelname)s: %(message)s') diff --git a/libmuscle/python/libmuscle/manager/test/conftest.py b/libmuscle/python/libmuscle/manager/test/conftest.py index fa981af1..8d81f560 100644 --- a/libmuscle/python/libmuscle/manager/test/conftest.py +++ b/libmuscle/python/libmuscle/manager/test/conftest.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from ymmsl import (Component, Conduit, Configuration, Model, Reference, Settings) @@ -9,8 +11,8 @@ @pytest.fixture -def logger(): - test_logger = Logger() +def logger(tmpdir): + test_logger = Logger(Path(str(tmpdir))) yield test_logger test_logger.close() diff --git a/libmuscle/python/libmuscle/manager/test/test_logger.py b/libmuscle/python/libmuscle/manager/test/test_logger.py index 01f1166b..aed3f46a 100644 --- a/libmuscle/python/libmuscle/manager/test/test_logger.py +++ b/libmuscle/python/libmuscle/manager/test/test_logger.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from libmuscle.logging import LogLevel, Timestamp from libmuscle.manager.logger import Logger @@ -9,8 +10,8 @@ def test_log_level(): assert LogLevel.CRITICAL.value > LogLevel.DEBUG.value -def test_create_logger(): - logger = Logger() +def test_create_logger(tmpdir): + logger = Logger(Path(str(tmpdir))) logger.close() From 19e01cad85b88ecbd7c66d8c7d928e38d3882a3e Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 12 Jan 2021 13:42:47 +0100 Subject: [PATCH 35/42] Fix time_stamp logging issue --- libmuscle/python/libmuscle/manager/logger.py | 31 ++++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index 2f9fa817..f948864f 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -6,6 +6,33 @@ from libmuscle.util import extract_log_file_location +class Formatter(logging.Formatter): + """A custom formatter that can format remote messages.""" + def usesTime(self) -> bool: + """Tells the formatter to make asctime available.""" + return True + + def formatMessage(self, record: logging.LogRecord) -> str: + """Formats a message for a record. + + If the record contains a time_stamp attribute, assumes that it + is a remote record and formats accordingly, otherwise formats + as a local record. + + Args: + record: The LogRecord to format. + + Returns: + The formatted message. + """ + if 'time_stamp' in record.__dict__: + return ( + '%(asctime)s %(name)s [%(time_stamp)-15s]' + ' %(levelname)s: %(message)s' % record.__dict__) + return ('%(asctime)s muscle_manager %(levelname)s: %(message)s' % + record.__dict__) + + class Logger: """The MUSCLE 3 Manager Logger component. @@ -20,9 +47,7 @@ def __init__(self, log_dir: Optional[Path] = None) -> None: log_dir = Path.cwd() logfile = extract_log_file_location(log_dir, 'muscle3_manager.log') self._local_handler = logging.FileHandler(str(logfile), mode='w') - formatter = logging.Formatter('%(time_stamp)-15s: %(name)s' - ' %(levelname)s: %(message)s') - self._local_handler.setFormatter(formatter) + self._local_handler.setFormatter(Formatter()) # Find and remove default handler to disable automatic console output # Testing for 'stderr' in the stringified version is not nice, but From 6c3d6e0ed537442ca59bed5e3e963a0faa52f0c3 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 22:54:34 +0100 Subject: [PATCH 36/42] Add temporary work-around for Configuration type issue --- muscle_manager/muscle_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py index a935186a..acf4b71e 100644 --- a/muscle_manager/muscle_manager.py +++ b/muscle_manager/muscle_manager.py @@ -17,6 +17,9 @@ def manage_simulation(ymmsl_files: Sequence[str]) -> None: for path in ymmsl_files: with open(path, 'r') as f: configuration.update(ymmsl.load(f)) + # temporary dummy data to satisfy type + configuration.update(PartialConfiguration( + None, None, {'dummy': None}, {'dummy': None})) server = start_server(configuration.as_configuration()) print(server.get_location()) From 351b551400ccb22caf5a8a3bf058853f0609af73 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:08:45 +0100 Subject: [PATCH 37/42] Switch back to released ymmsl/YAtiML dependencies --- .github/workflows/ci.yaml | 2 -- .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.5.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 2 +- .github/workflows/ci_python3.6.yaml | 2 +- .github/workflows/ci_python3.7.yaml | 2 +- .github/workflows/ci_python3.8.yaml | 2 +- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- .github/workflows/ci_ubuntu20.10.yaml | 2 +- setup.py | 2 +- 12 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 93e88b85..d3818f6e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,8 +18,6 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev - pip install git+https://github.com/yatiml/yatiml.git@develop - pip install git+https://github.com/multiscale/ymmsl-python.git@develop - name: Build and run the test suite env: diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index b514b9f6..3e999d4a 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 3e1177b0..79166e12 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index d3be4c23..d0415192 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6.1-eggs2 - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install -U git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && make test_python_only' diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml index 78605b61..6953054b 100644 --- a/.github/workflows/ci_python3.6.yaml +++ b/.github/workflows/ci_python3.6.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6-eggs - name: Run Python tests on 3.6 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && make test_python_only' diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml index d48d7bb3..318586e6 100644 --- a/.github/workflows/ci_python3.7.yaml +++ b/.github/workflows/ci_python3.7.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.7-eggs - name: Run Python tests on 3.7 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && make test_python_only' diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml index 9608db8b..870425cf 100644 --- a/.github/workflows/ci_python3.8.yaml +++ b/.github/workflows/ci_python3.8.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.8-eggs - name: Run Python tests on 3.8 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && make test_python_only' diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index b757d1c7..ba8ebc4e 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 7502bfe9..3afcdf98 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index f150e21a..9d1587e5 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip install -U git+https://github.com/yatiml/yatiml.git@develop && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml index d2ce7f89..7271cb8a 100644 --- a/.github/workflows/ci_ubuntu20.10.yaml +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install git+https://github.com/multiscale/ymmsl-python.git@develop && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' diff --git a/setup.py b/setup.py index 9ae50380..824e96ad 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ 'numpy>=1.12,<1.20', 'protobuf>=3.10.0, <4', 'typing_extensions', - 'ymmsl' # Also in CI, update there as well + 'ymmsl>=0.11.0' # Also in CI, update there as well ], setup_requires=[ 'pytest-runner', From ea2354e2b06cfc1e5415824cae1b5ef3267ab0d9 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:26:24 +0100 Subject: [PATCH 38/42] Fix Configuration work-around --- muscle_manager/muscle_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py index acf4b71e..73d3cbbc 100644 --- a/muscle_manager/muscle_manager.py +++ b/muscle_manager/muscle_manager.py @@ -2,7 +2,7 @@ import click import ymmsl -from ymmsl import PartialConfiguration +from ymmsl import Implementation, PartialConfiguration, Reference, Resources from libmuscle.manager.manager import start_server @@ -19,7 +19,9 @@ def manage_simulation(ymmsl_files: Sequence[str]) -> None: configuration.update(ymmsl.load(f)) # temporary dummy data to satisfy type configuration.update(PartialConfiguration( - None, None, {'dummy': None}, {'dummy': None})) + None, None, + {Reference('dummy'): Implementation(Reference('dummy'), '')}, + {Reference('dummy'): Resources(Reference('dummy'), 0)})) server = start_server(configuration.as_configuration()) print(server.get_location()) From b0de421740fa5b6cfe30d08ab57364ec798eb2c8 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:30:44 +0100 Subject: [PATCH 39/42] Add version 0.4.0 to the change log --- CHANGELOG.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2994d239..ed1b6927 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,28 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `_. +0.4.0 +***** + +Incompatible changes +-------------------- + +* `compute_elements` are now called `components` in .ymmsl files + +Improved +-------- + +* Use latest OpenSSL library when installing it automatically + +Fixed +----- + +* Handling of non-contiguous and F-order numpy arrays +* C++ memory usage for large dicts/lists now more reasonable +* Improved shutdown when Python submodel crashes +* Logging warning message + + 0.3.2 ***** From cd2332e524d0d3999c1204e9b217d84f50f11d36 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:38:02 +0100 Subject: [PATCH 40/42] Try to fix CI --- .github/workflows/ci.yaml | 1 + .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.5.yaml | 2 +- .github/workflows/ci_python3.6.1.yaml | 4 ++-- .github/workflows/ci_python3.6.yaml | 2 +- .github/workflows/ci_python3.7.yaml | 2 +- .github/workflows/ci_python3.8.yaml | 2 +- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- .github/workflows/ci_ubuntu20.10.yaml | 2 +- 11 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d3818f6e..45d2b554 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,6 +18,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev + pip install ymmsl==0.11.0 - name: Build and run the test suite env: diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 3e999d4a..32c4b1d7 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 79166e12..5f310a8e 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy" && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index d0415192..da7822a4 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}/.eggs - key: python-compatibility-3.6.1-eggs2 + key: python-compatibility-3.6.1-eggs - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml index 6953054b..09478248 100644 --- a/.github/workflows/ci_python3.6.yaml +++ b/.github/workflows/ci_python3.6.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6-eggs - name: Run Python tests on 3.6 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml index 318586e6..b110f88f 100644 --- a/.github/workflows/ci_python3.7.yaml +++ b/.github/workflows/ci_python3.7.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.7-eggs - name: Run Python tests on 3.7 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml index 870425cf..93756697 100644 --- a/.github/workflows/ci_python3.8.yaml +++ b/.github/workflows/ci_python3.8.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.8-eggs - name: Run Python tests on 3.8 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index ba8ebc4e..716bb931 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 3afcdf98..0a226a83 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index 9d1587e5..0b4cb0da 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml index 7271cb8a..5c2db62c 100644 --- a/.github/workflows/ci_ubuntu20.10.yaml +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl git && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' From 610c8ee35e6a36a95eb95bfda3c8e759ffbe9ad0 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:45:42 +0100 Subject: [PATCH 41/42] Set release version to 0.4.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cae7e608..1d0ba9ea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.3-dev +0.4.0 From 6781f0aaf093b348a2b6eb48568c50674e7b3e24 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 Jan 2021 23:46:53 +0100 Subject: [PATCH 42/42] Update badges to point to master --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ee6e82b0..d5adf666 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,11 @@ .. image:: https://github.com/multiscale/muscle3/raw/develop/docs/source/muscle3_logo_readme.png :alt: MUSCLE 3 -.. image:: https://readthedocs.org/projects/muscle3/badge/?version=develop - :target: https://muscle3.readthedocs.io/en/develop/?badge=develop +.. image:: https://readthedocs.org/projects/muscle3/badge/?version=master + :target: https://muscle3.readthedocs.io/en/develop/?badge=master :alt: Documentation Build Status -.. image:: https://github.com/multiscale/muscle3/workflows/continuous_integration/badge.svg?branch=develop +.. image:: https://github.com/multiscale/muscle3/workflows/continuous_integration/badge.svg?branch=master :target: https://github.com/multiscale/muscle3/actions :alt: Build Status @@ -17,8 +17,8 @@ :target: https://www.codacy.com/app/LourensVeen/muscle3 :alt: Codacy Grade -.. image:: https://requires.io/github/multiscale/muscle3/requirements.svg?branch=develop - :target: https://requires.io/github/multiscale/muscle3/requirements/?branch=develop +.. image:: https://requires.io/github/multiscale/muscle3/requirements.svg?branch=master + :target: https://requires.io/github/multiscale/muscle3/requirements/?branch=master :alt: Requirements Status .. image:: https://zenodo.org/badge/122876985.svg