diff --git a/contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java b/contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java index fdf4abce4b9..4f0f6fd35a2 100644 --- a/contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java +++ b/contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java @@ -1,5 +1,9 @@ package org.matsim.application; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.matsim.api.core.v01.Scenario; @@ -7,11 +11,14 @@ import org.matsim.application.options.InputOptions; import org.matsim.application.options.OutputOptions; import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigAliases; +import org.matsim.core.config.ConfigGroup; import org.matsim.core.config.ConfigUtils; import org.matsim.core.scenario.ScenarioUtils; import org.matsim.core.utils.io.IOUtils; import picocli.CommandLine; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -21,8 +28,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; public class ApplicationUtils { @@ -71,7 +80,128 @@ public static boolean isRunFromDesktop() { } /** - * Extends a context (usually config location) with an relative filename. + * Apply run configuration in yaml format. + */ + public static void applyConfigUpdate(Config config, Path yaml) { + + if (!Files.exists(yaml)) { + throw new IllegalArgumentException("Desired run config does not exist:" + yaml); + } + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory() + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); + + ConfigAliases aliases = new ConfigAliases(); + Deque emptyStack = new ArrayDeque<>(); + + try (BufferedReader reader = Files.newBufferedReader(yaml)) { + + JsonNode node = mapper.readTree(reader); + + Iterator> fields = node.fields(); + + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String configGroupName = aliases.resolveAlias(field.getKey(), emptyStack); + ConfigGroup group = config.getModules().get(configGroupName); + if (group == null) { + log.warn("Config group not found: {}", configGroupName); + continue; + } + + applyNodeToConfigGroup(field.getValue(), group); + } + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + } + + /** + * Sets the json config into + */ + private static void applyNodeToConfigGroup(JsonNode node, ConfigGroup group) { + + Iterator> fields = node.fields(); + + while (fields.hasNext()) { + Map.Entry field = fields.next(); + + if (isParameterSet(field.getValue())) { + + // store the current parameters sets, newly added sets are not merged with each other + List params = new ArrayList<>(group.getParameterSets(field.getKey())); + + for (JsonNode item : field.getValue()) { + applyNodeAsParameterSet(field.getKey(), item, group, params); + } + } else { + + if (field.getValue().isTextual()) + group.addParam(field.getKey(), field.getValue().textValue()); + else if (field.getValue().isArray()) { + // arrays are joined using "," + Stream stream = StreamSupport.stream(field.getValue().spliterator(), false); + String string = stream.map(n -> n.isTextual() ? n.textValue() : n.toString()).collect(Collectors.joining(",")); + group.addParam(field.getKey(), string); + } else + group.addParam(field.getKey(), field.getValue().toString()); + } + } + } + + /** + * Any array of complex object can be considered a config group. + */ + private static boolean isParameterSet(JsonNode node) { + + if (!node.isArray()) + return false; + + // any object can be assigned as parameter set + for (JsonNode el : node) { + if (!el.isObject()) + return false; + } + + return true; + } + + /** + * Handle possible update and creation of parameter sets within a config group. + */ + private static void applyNodeAsParameterSet(String groupName, JsonNode item, ConfigGroup group, List params) { + + Iterator> it = item.fields(); + while (!params.isEmpty() && it.hasNext()) { + + Map.Entry attr = it.next(); + List candidates = params.stream() + .filter(p -> p.getParams().containsKey(attr.getKey())) + .filter(p -> p.getParams().get(attr.getKey()) + .equals(attr.getValue().isTextual() ? attr.getValue().textValue() : attr.getValue().toString())) + .toList(); + + if (candidates.isEmpty()) + break; + + params = candidates; + } + + if (params.size() > 1) { + throw new IllegalArgumentException("Ambiguous parameter set: " + item); + } else if (params.size() == 1) { + applyNodeToConfigGroup(item, params.get(0)); + } else { + ConfigGroup newGroup = group.createParameterSet(groupName); + group.addParameterSet(newGroup); + applyNodeToConfigGroup(item, newGroup); + } + } + + /** + * Extends a context (usually config location) with a relative filename. * If the results is a local file, the path will be returned. Otherwise, it will be an url. * The results can be used as input for command line parameter or {@link IOUtils#resolveFileOrResource(String)}. * @@ -100,10 +230,11 @@ public static Path globFile(Path path, String pattern) { PathMatcher m = path.getFileSystem().getPathMatcher("glob:" + pattern); try { - return Files.list(path) - .filter(p -> m.matches(p.getFileName())) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No " + pattern + " file found.")); + try (Stream list = Files.list(path)) { + return list.filter(p -> m.matches(p.getFileName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No " + pattern + " file found.")); + } } catch (IOException e) { throw new RuntimeException(e); } diff --git a/contribs/application/src/main/java/org/matsim/application/MATSimApplication.java b/contribs/application/src/main/java/org/matsim/application/MATSimApplication.java index 5e2d86e7b2e..97722ab2e93 100644 --- a/contribs/application/src/main/java/org/matsim/application/MATSimApplication.java +++ b/contribs/application/src/main/java/org/matsim/application/MATSimApplication.java @@ -1,9 +1,5 @@ package org.matsim.application; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -11,7 +7,6 @@ import org.matsim.application.commands.RunScenario; import org.matsim.application.commands.ShowGUI; import org.matsim.core.config.Config; -import org.matsim.core.config.ConfigAliases; import org.matsim.core.config.ConfigGroup; import org.matsim.core.config.ConfigUtils; import org.matsim.core.config.groups.ControllerConfigGroup; @@ -23,15 +18,12 @@ import picocli.CommandLine; import javax.annotation.Nullable; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.Field; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.concurrent.Callable; @@ -170,7 +162,7 @@ public Integer call() throws Exception { Objects.requireNonNull(config); if (specs != null) - applySpecs(config, specs); + ApplicationUtils.applyConfigUpdate(config, specs); if (remainingArgs != null) { String[] args = remainingArgs.stream().map(s -> s.replace("-c:", "--config:")).toArray(String[]::new); @@ -219,90 +211,6 @@ public Integer call() throws Exception { return 0; } - /** - * Apply given specs to config. - */ - private static void applySpecs(Config config, Path specs) { - - if (!Files.exists(specs)) { - throw new IllegalArgumentException("Desired run config does not exist:" + specs); - } - - ObjectMapper mapper = new ObjectMapper(new YAMLFactory() - .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); - - ConfigAliases aliases = new ConfigAliases(); - Deque emptyStack = new ArrayDeque<>(); - - try (BufferedReader reader = Files.newBufferedReader(specs)) { - - JsonNode node = mapper.readTree(reader); - - Iterator> fields = node.fields(); - - while (fields.hasNext()) { - Map.Entry field = fields.next(); - String configGroupName = aliases.resolveAlias(field.getKey(), emptyStack); - ConfigGroup group = config.getModules().get(configGroupName); - if (group == null) { - log.warn("Config group not found: {}", configGroupName); - continue; - } - - applyNodeToConfigGroup(field.getValue(), group); - } - - } catch (IOException e) { - e.printStackTrace(); - } - - } - - /** - * Sets the json config into - */ - private static void applyNodeToConfigGroup(JsonNode node, ConfigGroup group) { - - Iterator> fields = node.fields(); - - while (fields.hasNext()) { - Map.Entry field = fields.next(); - - if (field.getValue().isArray()) { - - Collection params = group.getParameterSets(field.getKey()); - - // single node and entry merged directly - if (field.getValue().size() == 1 && params.size() == 1) { - applyNodeToConfigGroup(field.getValue().get(0), params.iterator().next()); - } else { - - for (JsonNode item : field.getValue()) { - - Map.Entry first = item.fields().next(); - Optional m = params.stream().filter(p -> p.getParams().get(first.getKey()).equals(first.getValue().textValue())).findFirst(); - if (m.isEmpty()) - throw new IllegalArgumentException("Could not find matching param by key" + first); - - applyNodeToConfigGroup(item, m.get()); - } - } - - } else { - - if (!field.getValue().isValueNode()) - throw new IllegalArgumentException("Received complex value type instead of primitive: " + field.getValue()); - - - if (field.getValue().isTextual()) - group.addParam(field.getKey(), field.getValue().textValue()); - else - group.addParam(field.getKey(), field.getValue().toString()); - } - } - } - - /** * Custom module configs that will be added to the {@link Config} object. * @@ -530,7 +438,7 @@ public static Controler prepare(MATSimApplication app, Config config, String... config = tmp != null ? tmp : config; if (app.specs != null) { - applySpecs(config, app.specs); + ApplicationUtils.applyConfigUpdate(config, app.specs); } if (app.remainingArgs != null) { diff --git a/contribs/application/src/test/java/org/matsim/application/ConfigYamlUpdateTest.java b/contribs/application/src/test/java/org/matsim/application/ConfigYamlUpdateTest.java new file mode 100644 index 00000000000..7a9394bb760 --- /dev/null +++ b/contribs/application/src/test/java/org/matsim/application/ConfigYamlUpdateTest.java @@ -0,0 +1,160 @@ +package org.matsim.application; + + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigGroup; +import org.matsim.core.config.ConfigUtils; +import org.matsim.core.config.ReflectiveConfigGroup; +import org.matsim.testcases.MatsimTestUtils; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class ConfigYamlUpdateTest { + + @RegisterExtension + private MatsimTestUtils utils = new MatsimTestUtils(); + + @Test + void standard() { + + Config config = ConfigUtils.createConfig(); + Path input = Path.of(utils.getClassInputDirectory()); + + ApplicationUtils.applyConfigUpdate( + config, input.resolve("standard.yml") + ); + + assertThat(config.controller().getRunId()).isEqualTo("567"); + assertThat(config.global().getNumberOfThreads()).isEqualTo(8); + + assertThat(config.scoring().getOrCreateModeParams("car").getConstant()).isEqualTo(-1); + assertThat(config.scoring().getOrCreateModeParams("bike").getConstant()).isEqualTo(-2); + } + + @Test + void createParamSet() { + + Config config = ConfigUtils.createConfig(); + Path input = Path.of(utils.getClassInputDirectory()); + + TestConfigGroup testGroup = ConfigUtils.addOrGetModule(config, TestConfigGroup.class); + + ApplicationUtils.applyConfigUpdate( + config, input.resolve("multiLevel.yml") + ); + + Collection params = testGroup.getParameterSets("params"); + + assertThat(params).hasSize(2); + + Iterator it = params.iterator(); + ConfigGroup next = it.next(); + + assertThat(next.getParams().get("mode")).isEqualTo("car"); + assertThat(next.getParams().get("values")).isEqualTo("-1, -2"); + + next = it.next(); + + assertThat(next.getParams().get("mode")).isEqualTo("bike"); + assertThat(next.getParams().get("values")).isEqualTo("3, 4"); + assertThat(next.getParams().get("extra")).isEqualTo("extra"); + } + + + @Test + void multiLevel() { + + Config config = ConfigUtils.createConfig(); + Path input = Path.of(utils.getClassInputDirectory()); + + TestConfigGroup testGroup = ConfigUtils.addOrGetModule(config, TestConfigGroup.class); + + testGroup.addParameterSet(new TestParamSet("car", "person", "work")); + testGroup.addParameterSet(new TestParamSet("bike", "person", "work")); + + ApplicationUtils.applyConfigUpdate( + config, input.resolve("multiLevel.yml") + ); + + Collection params = testGroup.getParameterSets("params"); + assertThat(params).hasSize(2); + + Iterator it = params.iterator(); + ConfigGroup next = it.next(); + + // These parameters are recognized as lists correctly + assertThat(next.getParams().get("values")).isEqualTo("-1, -2"); + + next = it.next(); + assertThat(next.getParams().get("values")).isEqualTo("3, 4"); + assertThat(next.getParams().get("extra")).isEqualTo("extra"); + + } + + @Test + void ambiguous() { + + Config config = ConfigUtils.createConfig(); + Path input = Path.of(utils.getClassInputDirectory()); + + TestConfigGroup testGroup = ConfigUtils.addOrGetModule(config, TestConfigGroup.class); + + testGroup.addParameterSet(new TestParamSet("car", "person", "work")); + testGroup.addParameterSet(new TestParamSet("car", "person", "home")); + + // This should fail because the parameter set is ambiguous + assertThatExceptionOfType(IllegalArgumentException.class).isThrownBy(() -> { + ApplicationUtils.applyConfigUpdate( + config, input.resolve("multiLevel.yml") + ); + }); + } + + + public static final class TestConfigGroup extends ReflectiveConfigGroup { + + @Parameter + private List values; + + public TestConfigGroup() { + super("test"); + } + + @Override + public ConfigGroup createParameterSet(String type) { + if (type.equals("params")) { + return new TestParamSet(); + } + + return super.createParameterSet(type); + } + } + + + public static final class TestParamSet extends ReflectiveConfigGroup { + + @Parameter + private List values; + + public TestParamSet() { + super("params", true); + } + + public TestParamSet(String mode, String subpopulation, String activity) { + super("params", true); + + this.addParam("mode", mode); + this.addParam("subpopulation", subpopulation); + this.addParam("activity", activity); + } + } + +} diff --git a/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/multiLevel.yml b/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/multiLevel.yml new file mode 100644 index 00000000000..63e40d29854 --- /dev/null +++ b/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/multiLevel.yml @@ -0,0 +1,10 @@ + +test: + params: + - mode: car + subpopulation: person + values: ["-1", "-2"] + - mode: bike + subpopulation: person + values: [3, 4] + extra: "extra" \ No newline at end of file diff --git a/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/standard.yml b/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/standard.yml new file mode 100644 index 00000000000..dc25f14bdf3 --- /dev/null +++ b/contribs/application/test/input/org/matsim/application/ConfigYamlUpdateTest/standard.yml @@ -0,0 +1,14 @@ +controler: + runId: 567 + +global: + numberOfThreads: 8 + +planCalcScore: + + scoringParameters: + - modeParams: + - mode: car + constant: -1 + - mode: bike + constant: -2 \ No newline at end of file