Skip to content

Commit

Permalink
improve config update from yaml and made it public
Browse files Browse the repository at this point in the history
  • Loading branch information
@rakow
rakow committed Feb 10, 2024
1 parent 13fc889 commit b0d7365
Show file tree
Hide file tree
Showing 5 changed files with 323 additions and 100 deletions.
143 changes: 137 additions & 6 deletions contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
package org.matsim.application;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.matsim.api.core.v01.Scenario;
import org.matsim.application.options.CrsOptions;
import org.matsim.application.options.InputOptions;
import org.matsim.application.options.OutputOptions;
import org.matsim.core.config.Config;
import org.matsim.core.config.ConfigAliases;
import org.matsim.core.config.ConfigGroup;
import org.matsim.core.config.ConfigUtils;
import org.matsim.core.scenario.ScenarioUtils;
import org.matsim.core.utils.io.IOUtils;
import picocli.CommandLine;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
Expand All @@ -21,8 +28,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

public class ApplicationUtils {

Expand Down Expand Up @@ -71,7 +80,128 @@ public static boolean isRunFromDesktop() {
}

/**
* Extends a context (usually config location) with an relative filename.
* Apply run configuration in yaml format.
*/
public static void applyConfigUpdate(Config config, Path yaml) {

if (!Files.exists(yaml)) {
throw new IllegalArgumentException("Desired run config does not exist:" + yaml);
}

ObjectMapper mapper = new ObjectMapper(new YAMLFactory()
.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES));

ConfigAliases aliases = new ConfigAliases();
Deque<String> emptyStack = new ArrayDeque<>();

try (BufferedReader reader = Files.newBufferedReader(yaml)) {

JsonNode node = mapper.readTree(reader);

Iterator<Map.Entry<String, JsonNode>> fields = node.fields();

while (fields.hasNext()) {
Map.Entry<String, JsonNode> field = fields.next();
String configGroupName = aliases.resolveAlias(field.getKey(), emptyStack);
ConfigGroup group = config.getModules().get(configGroupName);
if (group == null) {
log.warn("Config group not found: {}", configGroupName);
continue;
}

applyNodeToConfigGroup(field.getValue(), group);
}

} catch (IOException e) {
throw new UncheckedIOException(e);
}

}

/**
* Sets the json config into
*/
private static void applyNodeToConfigGroup(JsonNode node, ConfigGroup group) {

Iterator<Map.Entry<String, JsonNode>> fields = node.fields();

while (fields.hasNext()) {
Map.Entry<String, JsonNode> field = fields.next();

if (isParameterSet(field.getValue())) {

// store the current parameters sets, newly added sets are not merged with each other
List<? extends ConfigGroup> params = new ArrayList<>(group.getParameterSets(field.getKey()));

for (JsonNode item : field.getValue()) {
applyNodeAsParameterSet(field.getKey(), item, group, params);
}
} else {

if (field.getValue().isTextual())
group.addParam(field.getKey(), field.getValue().textValue());
else if (field.getValue().isArray()) {
// arrays are joined using ","
Stream<JsonNode> stream = StreamSupport.stream(field.getValue().spliterator(), false);
String string = stream.map(n -> n.isTextual() ? n.textValue() : n.toString()).collect(Collectors.joining(","));
group.addParam(field.getKey(), string);
} else
group.addParam(field.getKey(), field.getValue().toString());
}
}
}

/**
* Any array of complex object can be considered a config group.
*/
private static boolean isParameterSet(JsonNode node) {

if (!node.isArray())
return false;

// any object can be assigned as parameter set
for (JsonNode el : node) {
if (!el.isObject())
return false;
}

return true;
}

/**
* Handle possible update and creation of parameter sets within a config group.
*/
private static void applyNodeAsParameterSet(String groupName, JsonNode item, ConfigGroup group, List<? extends ConfigGroup> params) {

Iterator<Map.Entry<String, JsonNode>> it = item.fields();
while (!params.isEmpty() && it.hasNext()) {

Map.Entry<String, JsonNode> attr = it.next();
List<? extends ConfigGroup> candidates = params.stream()
.filter(p -> p.getParams().containsKey(attr.getKey()))
.filter(p -> p.getParams().get(attr.getKey())
.equals(attr.getValue().isTextual() ? attr.getValue().textValue() : attr.getValue().toString()))
.toList();

if (candidates.isEmpty())
break;

params = candidates;
}

if (params.size() > 1) {
throw new IllegalArgumentException("Ambiguous parameter set: " + item);
} else if (params.size() == 1) {
applyNodeToConfigGroup(item, params.get(0));
} else {
ConfigGroup newGroup = group.createParameterSet(groupName);
group.addParameterSet(newGroup);
applyNodeToConfigGroup(item, newGroup);
}
}

/**
* Extends a context (usually config location) with a relative filename.
* If the results is a local file, the path will be returned. Otherwise, it will be an url.
* The results can be used as input for command line parameter or {@link IOUtils#resolveFileOrResource(String)}.
*
Expand Down Expand Up @@ -100,10 +230,11 @@ public static Path globFile(Path path, String pattern) {
PathMatcher m = path.getFileSystem().getPathMatcher("glob:" + pattern);

try {
return Files.list(path)
.filter(p -> m.matches(p.getFileName()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("No " + pattern + " file found."));
try (Stream<Path> list = Files.list(path)) {
return list.filter(p -> m.matches(p.getFileName()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("No " + pattern + " file found."));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down
96 changes: 2 additions & 94 deletions contribs/application/src/main/java/org/matsim/application/MATSimApplication.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,12 @@
package org.matsim.application;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.matsim.api.core.v01.Scenario;
import org.matsim.application.commands.RunScenario;
import org.matsim.application.commands.ShowGUI;
import org.matsim.core.config.Config;
import org.matsim.core.config.ConfigAliases;
import org.matsim.core.config.ConfigGroup;
import org.matsim.core.config.ConfigUtils;
import org.matsim.core.config.groups.ControllerConfigGroup;
Expand All @@ -23,15 +18,12 @@
import picocli.CommandLine;

import javax.annotation.Nullable;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.Callable;
Expand Down Expand Up @@ -170,7 +162,7 @@ public Integer call() throws Exception {
Objects.requireNonNull(config);

if (specs != null)
applySpecs(config, specs);
ApplicationUtils.applyConfigUpdate(config, specs);

if (remainingArgs != null) {
String[] args = remainingArgs.stream().map(s -> s.replace("-c:", "--config:")).toArray(String[]::new);
Expand Down Expand Up @@ -219,90 +211,6 @@ public Integer call() throws Exception {
return 0;
}

/**
* Apply given specs to config.
*/
private static void applySpecs(Config config, Path specs) {

if (!Files.exists(specs)) {
throw new IllegalArgumentException("Desired run config does not exist:" + specs);
}

ObjectMapper mapper = new ObjectMapper(new YAMLFactory()
.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES));

ConfigAliases aliases = new ConfigAliases();
Deque<String> emptyStack = new ArrayDeque<>();

try (BufferedReader reader = Files.newBufferedReader(specs)) {

JsonNode node = mapper.readTree(reader);

Iterator<Map.Entry<String, JsonNode>> fields = node.fields();

while (fields.hasNext()) {
Map.Entry<String, JsonNode> field = fields.next();
String configGroupName = aliases.resolveAlias(field.getKey(), emptyStack);
ConfigGroup group = config.getModules().get(configGroupName);
if (group == null) {
log.warn("Config group not found: {}", configGroupName);
continue;
}

applyNodeToConfigGroup(field.getValue(), group);
}

} catch (IOException e) {
e.printStackTrace();
}

}

/**
* Sets the json config into
*/
private static void applyNodeToConfigGroup(JsonNode node, ConfigGroup group) {

Iterator<Map.Entry<String, JsonNode>> fields = node.fields();

while (fields.hasNext()) {
Map.Entry<String, JsonNode> field = fields.next();

if (field.getValue().isArray()) {

Collection<? extends ConfigGroup> params = group.getParameterSets(field.getKey());

// single node and entry merged directly
if (field.getValue().size() == 1 && params.size() == 1) {
applyNodeToConfigGroup(field.getValue().get(0), params.iterator().next());
} else {

for (JsonNode item : field.getValue()) {

Map.Entry<String, JsonNode> first = item.fields().next();
Optional<? extends ConfigGroup> m = params.stream().filter(p -> p.getParams().get(first.getKey()).equals(first.getValue().textValue())).findFirst();
if (m.isEmpty())
throw new IllegalArgumentException("Could not find matching param by key" + first);

applyNodeToConfigGroup(item, m.get());
}
}

} else {

if (!field.getValue().isValueNode())
throw new IllegalArgumentException("Received complex value type instead of primitive: " + field.getValue());


if (field.getValue().isTextual())
group.addParam(field.getKey(), field.getValue().textValue());
else
group.addParam(field.getKey(), field.getValue().toString());
}
}
}


/**
* Custom module configs that will be added to the {@link Config} object.
*
Expand Down Expand Up @@ -530,7 +438,7 @@ public static Controler prepare(MATSimApplication app, Config config, String...
config = tmp != null ? tmp : config;

if (app.specs != null) {
applySpecs(config, app.specs);
ApplicationUtils.applyConfigUpdate(config, app.specs);
}

if (app.remainingArgs != null) {
Expand Down
Loading

0 comments on commit b0d7365

Please sign in to comment.