Fix connection parameter handling

trollmoves/dispatcher.py

@@ -277,7 +277,6 @@ def create_dest_url(self, msg, client, conf):
         config = self.config[client].copy()
         _verify_filepattern(config, msg)
         config.update(conf)
-        connection_parameters = config.get('connection_parameters')
 
         host = config['host']
 
@@ -289,7 +288,7 @@ def create_dest_url(self, msg, client, conf):
             metadata)
         parts = urlsplit(host)
         host_path = urlunsplit((parts.scheme, parts.netloc, path, parts.query, parts.fragment))
-        return host_path, connection_parameters, client
+        return host_path, config, client
 
     def _get_file_messages_from_dataset_message(self, msg):
         """From a dataset type message create individual messages for each file in the dataset."""

trollmoves/movers.py

@@ -483,13 +483,35 @@ class S3Mover(Mover):
     changing the filename. The new destination filename will be the last part
     of the provided destination following the last slash ('/').
 
+    In the Trollmoves Server config, which is in .ini format, the connection parameters
+    and other dictionary-like items can be defined with douple underscore format::
+
+        connection_parameters__secret = secret
+        connection_parameters__client_kwargs__endpoint_url = https://endpoint.url
+        connection_parameters__client_kwargs__verify = false
+
+    will result in a nested dictionary item::
+
+        {
+            'connection_parameters': {
+                'secret': 'secret',
+                'client_kwargs': {
+                    'endpoint_url': 'https://endpoint.url',
+                    'verify': False
+                }
+            }
+        }
+
+    Note that boolean values are converted. No handling for numeric values have been implemented!
+
     """
 
     def copy(self):
         """Copy the file to a bucket."""
         if S3FileSystem is None:
             raise ImportError("S3Mover requires 's3fs' to be installed.")
-        s3 = S3FileSystem(**self.attrs)
+        connection_parameters = self.attrs.get("connection_parameters", {})
+        s3 = S3FileSystem(**connection_parameters)
         destination_file_path = self._get_destination()
         LOGGER.debug('destination_file_path = %s', destination_file_path)
         _create_s3_destination_path(s3, destination_file_path)

trollmoves/server.py

@@ -584,6 +584,7 @@
         _parse_nameserver(res[section], cp_[section])
         _parse_addresses(res[section])
         _parse_delete(res[section], cp_[section])
+        res[section] = _create_config_sub_dicts(res[section])
         if not _check_origin_and_listen(res, section):
             continue
         if not _check_topic(res, section):
@@ -622,6 +623,34 @@
         conf["delete"] = val
 
 
+def _create_config_sub_dicts(original):
+    # Take a copy so we can modify the values if necessary
+    res = dict(original.items())
+    for key in original.keys():
+        parts = key.split("__")
+        if len(parts) > 1:
+            _create_dicts(res, parts, original[key])
+            del res[key]
+    return res
+
+
+def _create_dicts(res, parts, val):
+    cur = res
+    for part in parts[:-1]:
+        if part not in cur:
+            cur[part] = {}
+        cur = cur[part]
+    cur[parts[-1]] = _check_bool(val)
+
+
+def _check_bool(val):
+    if val.lower() in ["0", "false"]:
+        return False
+    elif val.lower() in ["1", "true"]:
+        return True
+    return val
+
+
 def _check_origin_and_listen(res, section):
     if ("origin" not in res[section]) and ('listen' not in res[section]):
         LOGGER.warning("Incomplete section %s: add an 'origin' or 'listen' item.", section)

trollmoves/tests/test_dispatcher.py

@@ -454,7 +454,7 @@ def test_get_destinations_single_destination(viirs_green_snow_message, dispatche
     assert len(res) == 1
     url, attrs, client = res[0]
     assert url == expected_url
-    assert attrs == expected_attrs
+    assert attrs["connection_parameters"] == expected_attrs
     assert client == "target1"
 
 
@@ -481,7 +481,7 @@ def _assert_get_destinations_res(res, expected_length, expected_url, expected_at
     assert len(res) == expected_length
     for i, (url, attrs, client) in enumerate(res):
         assert url == expected_url[i]
-        assert attrs == expected_attrs[i]
+        assert attrs["connection_parameters"] == expected_attrs[i]
         assert client == expected_client[i]
 
 
@@ -684,7 +684,7 @@ def test_create_dest_url_ssh_no_username(create_dest_url_message):
 
         expected_url = "ssh://server.target2.com/satellite/viirs/sat_201909190919_NOAA-20.tif"
         assert url == expected_url
-        assert params == {'ssh_key_filename': '~/.ssh/rsa_id.pub'}
+        assert params["connection_parameters"] == {'ssh_key_filename': '~/.ssh/rsa_id.pub'}
         assert client == "target2"
     finally:
         if dispatcher is not None:

trollmoves/tests/test_movers.py

@@ -173,14 +173,14 @@ def test_s3_copy_file_to_base_using_connection_parameters(S3FileSystem):
     """Test copying to base of S3 bucket."""
     # Get the connection parameters:
     config = yaml.safe_load(test_yaml_s3_connection_params)
-    attrs = config['target-s3-example1']['connection_parameters']
+    attrs = config['target-s3-example1']
 
     s3_mover = _get_s3_mover(ORIGIN, "s3://data-bucket/", **attrs)
-    assert s3_mover.attrs['client_kwargs'] == {'endpoint_url': 'https://minio-server.mydomain.se:9000',
-                                               'verify': False}
-    assert s3_mover.attrs['secret'] == 'my-super-secret-key'
-    assert s3_mover.attrs['key'] == 'my-access-key'
-    assert s3_mover.attrs['use_ssl'] is True
+    assert s3_mover.attrs['connection_parameters']['client_kwargs'] == {
+        'endpoint_url': 'https://minio-server.mydomain.se:9000', 'verify': False}
+    assert s3_mover.attrs['connection_parameters']['secret'] == 'my-super-secret-key'
+    assert s3_mover.attrs['connection_parameters']['key'] == 'my-access-key'
+    assert s3_mover.attrs['connection_parameters']['use_ssl'] is True
 
     s3_mover.copy()
 

trollmoves/tests/test_server.py

@@ -170,17 +170,40 @@ def test_empty_init_arguments_does_not_crash_add(self):
         Deleter(dict()).add('bla')
 
 
-config_file = b"""
+CONFIG_INI = b"""
 [eumetcast-hrit-0deg]
 origin = /local_disk/tellicast/received/MSGHRIT/H-000-{nominal_time:%Y%m%d%H%M}-{compressed:_<2s}
 request_port = 9094
 publisher_port = 9010
 info = sensor=seviri;variant=0DEG
 topic = /1b/hrit-segment/0deg
 delete = False
+connection_parameters__secret = secret
+connection_parameters__client_kwargs__endpoint_url = https://endpoint.url
+connection_parameters__client_kwargs__verify = false
 """
 
 
+def test_read_config_ini_with_dicts():
+    """Test reading a config in ini format when dictionary values should be created."""
+    from trollmoves.server import read_config
+
+    with NamedTemporaryFile(suffix=".ini") as config_file:
+        config_file.write(CONFIG_INI)
+        config_file.flush()
+        config = read_config(config_file.name)
+        eumetcast = config["eumetcast-hrit-0deg"]
+        assert "origin" in eumetcast
+        assert "request_port" in eumetcast
+        assert "publisher_port" in eumetcast
+        assert "info" in eumetcast
+        assert "topic" in eumetcast
+        assert "delete" in eumetcast
+        assert eumetcast["connection_parameters"]["secret"] == "secret"
+        assert eumetcast["connection_parameters"]["client_kwargs"]["endpoint_url"] == "https://endpoint.url"
+        assert eumetcast["connection_parameters"]["client_kwargs"]["verify"] is False
+
+
 class TestMoveItServer:
     """Test the move it server."""
 
@@ -195,7 +218,7 @@ def test_reloads_config_crashes_when_config_file_does_not_exist(self):
     def test_reloads_config_on_example_config(self, fake_publisher):
         """Test that config can be reloaded with basic example."""
         with NamedTemporaryFile() as temporary_config_file:
-            temporary_config_file.write(config_file)
+            temporary_config_file.write(CONFIG_INI)
             config_filename = temporary_config_file.name
             cmd_args = parse_args(["--port", "9999", config_filename])
             server = MoveItServer(cmd_args)
@@ -206,7 +229,7 @@ def test_reloads_config_on_example_config(self, fake_publisher):
     def test_reloads_config_calls_reload_config(self, mock_reload_config, mock_publisher):
         """Test that config file can be reloaded."""
         with NamedTemporaryFile() as temporary_config_file:
-            temporary_config_file.write(config_file)
+            temporary_config_file.write(CONFIG_INI)
             config_filename = temporary_config_file.name
             cmd_args = parse_args(["--port", "9999", config_filename])
             server = MoveItServer(cmd_args)
@@ -218,7 +241,7 @@ def test_reloads_config_calls_reload_config(self, mock_reload_config, mock_publi
     def test_signal_reloads_config_calls_reload_config(self, mock_reload_config, mock_publisher):
         """Test that config file can be reloaded through signal."""
         with NamedTemporaryFile() as temporary_config_file:
-            temporary_config_file.write(config_file)
+            temporary_config_file.write(CONFIG_INI)
             config_filename = temporary_config_file.name
             cmd_args = parse_args([config_filename])
             client = MoveItServer(cmd_args)