From dec39a539d71e97162ef1c969472a826439c7591 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 16:29:24 +0200 Subject: [PATCH 01/45] added the formatter feature --- src/org/rascalmpl/library/lang/json/IO.java | 34 ++++++++++++++++--- src/org/rascalmpl/library/lang/json/IO.rsc | 26 ++++++++++++-- .../lang/json/internal/JsonValueWriter.java | 21 ++++++++++++ 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 6cecd64debc..a44c09b83af 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -22,6 +22,7 @@ import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.exceptions.Throw; import org.rascalmpl.library.lang.json.internal.IValueAdapter; import org.rascalmpl.library.lang.json.internal.JSONReadingTypeVisitor; import org.rascalmpl.library.lang.json.internal.JsonValueReader; @@ -29,14 +30,16 @@ import org.rascalmpl.types.TypeReifier; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; +import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.functions.IFunction; import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; +import io.usethesource.vallang.ISet; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; import io.usethesource.vallang.IValue; -import io.usethesource.vallang.IValueFactory; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeStore; @@ -49,10 +52,10 @@ import com.ibm.icu.text.DateFormat; public class IO { - private final IValueFactory values; + private final IRascalValueFactory values; private final IRascalMonitor monitor; - public IO(IValueFactory values, IRascalMonitor monitor) { + public IO(IRascalValueFactory values, IRascalMonitor monitor) { super(); this.values = values; this.monitor = monitor; @@ -137,7 +140,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool } } - public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, ISet formatters) { try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); @@ -154,8 +157,28 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations } } - public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, ISet formatters) { StringWriter string = new StringWriter(); + IFunction formatFunction = null; + + if (!formatters.isEmpty()) { + // here we construct a choice function (should be an IRascalValueFactory builder) + var first = formatters.iterator().next(); + formatFunction = values.function(first.getType(), (args, kwargs) -> { + Throw thrown = null; + for (IValue f : formatters) { + try { + return ((IFunction) f).call(args); + } + catch (Throw x) { + thrown = x; + // callfailed is to be expected + } + } + + throw thrown; + }); + } try (JsonWriter out = new JsonWriter(string)) { if (indent.intValue() > 0) { @@ -166,6 +189,7 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setFormatters(formatFunction) .write(out, value); return values.string(string.toString()); diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 463c8f299c4..505c349c2e7 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -57,7 +57,29 @@ java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\' If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z. If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation. } -java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true); +java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, set[JSONFormatter[value]] formatters = {}); @javaClass{org.rascalmpl.library.lang.json.IO} -java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true); +java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, set[JSONFormatter[value]] formatters = {}); + +@synopsis{((writeJSON)) and ((asJSON)) uses `Formatter` functions to flatten structured data to strings, on-demand} +@description{ +A JSONFormatter can be passed to the ((writeJSON)) and ((asJSON)) functions. When/if the type matches an algebraic data-type +to be serialized, then it is applied and the resulting string is serialized to the JSON stream instead of the structured data. + +The goal of JSONFormat and its dual JSONParser is to bridge the gap between string-based JSON encodings and typical +Rascal algebraic combinators. +} +alias JSONFormatter[&T] = str (&T); + +@synopsis{((readJSON)) and ((parseJSON)) use JSONParser functions to turn unstructured data into structured data.} +@description{ +A parser JSONParser can be passed to ((readJSON)) and ((parseJSON)). When the reader expects an algebraic data-type +or a syntax type, but the input at that moment is a JSON string, then the parser is called on that string (after string.trim()). + +The resulting data constructor is put into the resulting value instead of a normal string. + +The goal of JSONParser and its dual JSONFormatter is to bridge the gap between string-based JSON encodings and typical +Rascal algebraic combinators. +} +alias JSONParser[&T] = &T (str); \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index c20838b972f..b27431c5bde 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -17,6 +17,9 @@ import java.util.Map; import java.util.Map.Entry; +import org.rascalmpl.exceptions.Throw; +import org.rascalmpl.values.functions.IFunction; + import com.google.gson.stream.JsonWriter; import io.usethesource.vallang.IBool; @@ -44,6 +47,7 @@ public class JsonValueWriter { private boolean datesAsInts = true; private boolean unpackedLocations = false; private boolean dropOrigins = true; + private IFunction formatters; public JsonValueWriter() { setCalendarFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); @@ -78,6 +82,11 @@ public JsonValueWriter setDropOrigins(boolean setting) { return this; } + public JsonValueWriter setFormatters(IFunction formatters) { + this.formatters = formatters; + return this; + } + public void write(JsonWriter out, IValue value) throws IOException { value.accept(new IValueVisitor() { @@ -222,6 +231,18 @@ public Void visitNode(INode o) throws IOException { @Override public Void visitConstructor(IConstructor o) throws IOException { + if (formatters != null) { + try { + var formatted = formatters.call(o); + if (formatted != null) { + visitString((IString) formatted); + return null; + } + } + catch (Throw x) { + // it happens + } + } if (o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { // enums! out.value(o.getName()); From c4107bfbd7cb49b32c15648d118c4374f81dc8ed Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 17:20:19 +0200 Subject: [PATCH 02/45] also added dual parsing functionality for the reader/unfinished --- src/org/rascalmpl/library/lang/json/IO.java | 47 +++++++++---------- src/org/rascalmpl/library/lang/json/IO.rsc | 47 ++++++++++++------- .../lang/json/internal/JsonValueReader.java | 40 ++++++++++++++-- .../rascalmpl/semantics/dynamic/Module.java | 2 - ...ascalJUnitParallelRecursiveTestRunner.java | 1 - .../infrastructure/RascalJUnitTestRunner.java | 1 - .../rascalmpl/test/parser/StackNodeTest.java | 1 - 7 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index a44c09b83af..9919b79214b 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -22,11 +22,11 @@ import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; -import org.rascalmpl.exceptions.Throw; import org.rascalmpl.library.lang.json.internal.IValueAdapter; import org.rascalmpl.library.lang.json.internal.JSONReadingTypeVisitor; import org.rascalmpl.library.lang.json.internal.JsonValueReader; import org.rascalmpl.library.lang.json.internal.JsonValueWriter; +import org.rascalmpl.types.ReifiedType; import org.rascalmpl.types.TypeReifier; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; @@ -103,7 +103,7 @@ public IValue fromJSON(IValue type, IString src) { } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -111,6 +111,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) .read(in, start); } catch (IOException e) { @@ -122,14 +123,20 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, } } - public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isBottom()) { + // ignore the default parser + parsers = null; + } + try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) .read(in, start); } catch (IOException e) { @@ -140,7 +147,13 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool } } - public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, ISet formatters) { + public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IFunction formatter) { + if (formatter.getType().getFieldType(0).isTop()) { + // ignore default function + formatter = null; + } + + try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); @@ -151,33 +164,19 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setFormatters(formatter) .write(out, value); } catch (IOException e) { throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } } - public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, ISet formatters) { + public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IFunction formatter) { StringWriter string = new StringWriter(); - IFunction formatFunction = null; - - if (!formatters.isEmpty()) { - // here we construct a choice function (should be an IRascalValueFactory builder) - var first = formatters.iterator().next(); - formatFunction = values.function(first.getType(), (args, kwargs) -> { - Throw thrown = null; - for (IValue f : formatters) { - try { - return ((IFunction) f).call(args); - } - catch (Throw x) { - thrown = x; - // callfailed is to be expected - } - } - throw thrown; - }); + if (formatter.getType().getFieldType(0).isTop()) { + // ignore default function + formatter = null; } try (JsonWriter out = new JsonWriter(string)) { @@ -189,7 +188,7 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) - .setFormatters(formatFunction) + .setFormatters(formatter) .write(out, value); return values.string(string.toString()); diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 505c349c2e7..b503db12a79 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -35,32 +35,37 @@ public java &T fromJSON(type[&T] typ, str src); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{reads JSON values from a stream In general the translation behaves as follows: - * Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields) - * Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below) - * Booleans translate to bools - * If the expected type provided is a datetime then an int instant is mapped and if a string is found then the dateTimeFormat parameter will be used to configure the parsing of a date-time string - * If the expected type provided is an ADT then this reader will try to "parse" each object as a constructor for that ADT. It helps if there is only one constructor for that ADT. Positional parameters will be mapped by name as well as keyword parameters. - * If the expected type provided is a node then it will construct a node named "object" and map the fields to keyword fields. - * If num, int, real or rat are expected both strings and number values are mapped - * If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of - a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]}} -java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +* Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields) +* Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below) +* Booleans translate to bools +* If the expected type provided is a datetime then an int instant is mapped and if a string is found then the dateTimeFormat parameter will be used to configure the parsing of a date-time string +* If the expected type provided is an ADT then this reader will try to "parse" each object as a constructor for that ADT. It helps if there is only one constructor for that ADT. Positional parameters will be mapped by name as well as keyword parameters. +* If the expected type provided is a node then it will construct a node named "object" and map the fields to keyword fields. +* If num, int, real or rat are expected both strings and number values are mapped +* If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of + a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]} +* Go to ((JSONParser)) to find out how to use the optional `parsers` parameter. +} +java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[node] parser = (type[void] _, str _) { throw "default parser"; }); @javaClass{org.rascalmpl.library.lang.json.IO} -@synopsis{parses JSON values from a string +@synopsis{parses JSON values from a string. In general the translation behaves as the same as for ((readJSON)).} -java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[node] parser = (type[void] _, str _) { throw "default parser"; }); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{writes `val` to the location `target`} @description{ - If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z. - If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation. +* If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z. +* If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation. +* Check out ((JSONFormatter)) on how to use the `formatters` parameter + } -java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, set[JSONFormatter[value]] formatters = {}); +java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, JSONFormatter[value] formatter = str (value _) { fail; }); @javaClass{org.rascalmpl.library.lang.json.IO} -java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, set[JSONFormatter[value]] formatters = {}); +@synopsis{Does what ((writeJSON)) does but serializes to a string instead of a location target.} +java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, JSONFormatter[value] formatter = str (value _) { fail; }); @synopsis{((writeJSON)) and ((asJSON)) uses `Formatter` functions to flatten structured data to strings, on-demand} @description{ @@ -82,4 +87,12 @@ The resulting data constructor is put into the resulting value instead of a norm The goal of JSONParser and its dual JSONFormatter is to bridge the gap between string-based JSON encodings and typical Rascal algebraic combinators. } -alias JSONParser[&T] = &T (str); \ No newline at end of file +@benefits{ +* Use parsers to create more structure than JSON provides. +} +@pitfalls{ +* The `type[&T]` argument is called dynamically by the JSON reader; it does not contain the +grammar. It does encode the expected type of the parse result. +* The expected types can only be `data` types, not syntax types. +} +alias JSONParser[&T] = &T (type[&T], str); \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 23b64f81606..335ae34dc5e 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -20,6 +20,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -27,14 +28,18 @@ import java.util.Set; import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.exceptions.Throw; import org.rascalmpl.uri.URIUtil; +import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.functions.IFunction; + +import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.IMapWriter; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; -import io.usethesource.vallang.IValueFactory; import io.usethesource.vallang.io.StandardTextReader; import io.usethesource.vallang.type.ITypeVisitor; import io.usethesource.vallang.type.Type; @@ -52,19 +57,20 @@ public class JsonValueReader { private static final TypeFactory TF = TypeFactory.getInstance(); private final TypeStore store; - private final IValueFactory vf; + private final IRascalValueFactory vf; private ThreadLocal format; private final IRascalMonitor monitor; private ISourceLocation src; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + private IFunction parsers; /** * @param vf factory which will be used to construct values * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields */ - public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { + public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { this.vf = vf; this.store = store; this.monitor = monitor; @@ -87,7 +93,7 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor } } - public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { + public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { this(vf, new TypeStore(), monitor, src); } @@ -105,6 +111,11 @@ protected SimpleDateFormat initialValue() { return this; } + public JsonValueReader setParsers(IFunction parsers) { + this.parsers = parsers; + return this; + } + /** * Read and validate a Json stream as an IValue * @param in json stream @@ -481,8 +492,27 @@ private int getCol() { @Override public IValue visitAbstractData(Type type) throws IOException { if (in.peek() == JsonToken.STRING) { + var stringInput = in.nextString(); + + // might be a parsable string. let's see. + if (parsers != null) { + var symbol = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); + var reified = vf.reifiedType(symbol, vf.map()); + try { + return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); + } + catch (Throw t) { + Type excType = t.getException().getType(); + + if (excType.isAbstractData() && ((IConstructor) t.getException()).getConstructorType().getName().equals("ParseError")) { + throw new IOException(t); // an actual parse error is meaningful to report + } + // otherwise we fall through to enum recognition + } + } + // enum! - Set enumCons = store.lookupConstructor(type, in.nextString()); + Set enumCons = store.lookupConstructor(type, stringInput); for (Type candidate : enumCons) { if (candidate.getArity() == 0) { diff --git a/src/org/rascalmpl/semantics/dynamic/Module.java b/src/org/rascalmpl/semantics/dynamic/Module.java index 00b88c2cf80..8a58e7a6e3f 100644 --- a/src/org/rascalmpl/semantics/dynamic/Module.java +++ b/src/org/rascalmpl/semantics/dynamic/Module.java @@ -26,8 +26,6 @@ import org.rascalmpl.interpreter.result.Result; import org.rascalmpl.interpreter.result.ResultFactory; import org.rascalmpl.interpreter.utils.Names; -import org.rascalmpl.uri.URIUtil; - import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; diff --git a/src/org/rascalmpl/test/infrastructure/RascalJUnitParallelRecursiveTestRunner.java b/src/org/rascalmpl/test/infrastructure/RascalJUnitParallelRecursiveTestRunner.java index 77a199a134e..8cf30986dec 100644 --- a/src/org/rascalmpl/test/infrastructure/RascalJUnitParallelRecursiveTestRunner.java +++ b/src/org/rascalmpl/test/infrastructure/RascalJUnitParallelRecursiveTestRunner.java @@ -30,7 +30,6 @@ import org.junit.runner.notification.RunNotifier; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.ITestResultListener; -import org.rascalmpl.interpreter.NullRascalMonitor; import org.rascalmpl.interpreter.TestEvaluator; import org.rascalmpl.interpreter.env.GlobalEnvironment; import org.rascalmpl.interpreter.env.ModuleEnvironment; diff --git a/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java b/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java index 992a25366a5..a4826a66e70 100644 --- a/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java +++ b/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java @@ -28,7 +28,6 @@ import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.ITestResultListener; -import org.rascalmpl.interpreter.NullRascalMonitor; import org.rascalmpl.interpreter.TestEvaluator; import org.rascalmpl.interpreter.env.GlobalEnvironment; import org.rascalmpl.interpreter.env.ModuleEnvironment; diff --git a/test/org/rascalmpl/test/parser/StackNodeTest.java b/test/org/rascalmpl/test/parser/StackNodeTest.java index bfc52a4f5b8..6b849ca5fa7 100644 --- a/test/org/rascalmpl/test/parser/StackNodeTest.java +++ b/test/org/rascalmpl/test/parser/StackNodeTest.java @@ -3,7 +3,6 @@ import org.junit.Assert; import org.junit.Test; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; -import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; import org.rascalmpl.parser.gtd.stack.filter.follow.AtEndOfLineRequirement; From a0dbc27f4d9569b4276cfa698564548cfbbe8db1 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 17:42:58 +0200 Subject: [PATCH 03/45] parsing works now too --- src/org/rascalmpl/library/lang/json/IO.java | 7 ++++++- src/org/rascalmpl/library/lang/json/IO.rsc | 4 ++-- .../library/lang/json/internal/JsonValueReader.java | 11 ++++++++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 9919b79214b..60f90b2b407 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -107,6 +107,11 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { + // ignore the default parser + parsers = null; + } + try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) @@ -127,7 +132,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isBottom()) { + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { // ignore the default parser parsers = null; } diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index b503db12a79..740c14f4e2a 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -46,12 +46,12 @@ In general the translation behaves as follows: a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]} * Go to ((JSONParser)) to find out how to use the optional `parsers` parameter. } -java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[node] parser = (type[void] _, str _) { throw "default parser"; }); +java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{parses JSON values from a string. In general the translation behaves as the same as for ((readJSON)).} -java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[node] parser = (type[void] _, str _) { throw "default parser"; }); +java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{writes `val` to the location `target`} diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 335ae34dc5e..0e3679ce377 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -496,8 +496,8 @@ public IValue visitAbstractData(Type type) throws IOException { // might be a parsable string. let's see. if (parsers != null) { - var symbol = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); - var reified = vf.reifiedType(symbol, vf.map()); + var reified = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); + try { return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); } @@ -520,7 +520,12 @@ public IValue visitAbstractData(Type type) throws IOException { } } - throw new IOException("no nullary constructor found for " + type); + if (parsers != null) { + throw new IOException("parser failed to recognize \"" + stringInput + "\" and no nullary constructor found for " + type + "either"); + } + else { + throw new IOException("no nullary constructor found for " + type); + } } assert in.peek() == JsonToken.BEGIN_OBJECT; From 6ec3df2138a43d7e409b8a7631c7151018199517 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 17:50:11 +0200 Subject: [PATCH 04/45] all webservers can now unparse while streaming to json as well --- src/org/rascalmpl/library/Content.rsc | 3 ++- src/org/rascalmpl/library/lang/json/IO.java | 17 +---------------- .../lang/json/internal/JsonValueReader.java | 6 ++++++ .../lang/json/internal/JsonValueWriter.java | 5 +++++ src/org/rascalmpl/library/util/TermREPL.java | 2 ++ src/org/rascalmpl/library/util/Webserver.java | 2 ++ src/org/rascalmpl/repl/REPLContentServer.java | 3 +++ 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index 28be88bbe94..9ad69215703 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -2,6 +2,7 @@ @synopsis{Content provides access to the content server of the Rascal terminal for viewing interactive HTML output.} module Content +import lang::json::IO; @synopsis{Content wraps the HTTP Request/Response API to support interactive visualization types on the terminal ((RascalShell)).} @@ -80,7 +81,7 @@ which involves a handy, automatic, encoding of Rascal values into json values. data Response = response(Status status, str mimeType, map[str,str] header, str content) | fileResponse(loc file, str mimeType, map[str,str] header) - | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }) ; diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 60f90b2b407..76b7b4fad12 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -131,11 +131,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - - if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { - // ignore the default parser - parsers = null; - } + try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { in.setLenient(lenient.getValue()); @@ -153,12 +149,6 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool } public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IFunction formatter) { - if (formatter.getType().getFieldType(0).isTop()) { - // ignore default function - formatter = null; - } - - try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); @@ -179,11 +169,6 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IFunction formatter) { StringWriter string = new StringWriter(); - if (formatter.getType().getFieldType(0).isTop()) { - // ignore default function - formatter = null; - } - try (JsonWriter out = new JsonWriter(string)) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 0e3679ce377..d0bd83f90b2 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -29,6 +29,7 @@ import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.Throw; +import org.rascalmpl.types.ReifiedType; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; @@ -112,6 +113,11 @@ protected SimpleDateFormat initialValue() { } public JsonValueReader setParsers(IFunction parsers) { + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { + // ignore the default parser + parsers = null; + } + this.parsers = parsers; return this; } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index b27431c5bde..d72bf3790ed 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -83,6 +83,11 @@ public JsonValueWriter setDropOrigins(boolean setting) { } public JsonValueWriter setFormatters(IFunction formatters) { + if (formatters.getType().getFieldType(0).isTop()) { + // ignore default function + formatters = null; + } + this.formatters = formatters; return this; } diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index f149bb36340..61572137b19 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -238,9 +238,11 @@ private void handleJSONResponse(Map output, IConstructor re IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue formatters = kws.getParameter("formatter"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters((IFunction) formatters) .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 292241ed33a..88918e4c61c 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -238,9 +238,11 @@ private Response translateJsonResponse(Method method, IConstructor cons) { IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue formatters = kws.getParameter("formatter"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters((IFunction) formatters) .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); try { diff --git a/src/org/rascalmpl/repl/REPLContentServer.java b/src/org/rascalmpl/repl/REPLContentServer.java index de811bcb6ef..1fc0e24300e 100644 --- a/src/org/rascalmpl/repl/REPLContentServer.java +++ b/src/org/rascalmpl/repl/REPLContentServer.java @@ -16,6 +16,7 @@ import org.rascalmpl.library.lang.json.internal.JsonValueWriter; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.functions.IFunction; import com.google.gson.stream.JsonWriter; @@ -135,9 +136,11 @@ private static Response translateJsonResponse(Method method, IConstructor cons) IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue formatters = kws.getParameter("formatter"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters((IFunction) formatters) .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); try { From 0006042e06d3ae961fb63b95aaa540b4bddc8d43 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 19:54:17 +0200 Subject: [PATCH 05/45] added CytoStyle structure --- src/org/rascalmpl/library/Content.rsc | 2 +- src/org/rascalmpl/library/vis/Graphs.rsc | 33 ++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index 9ad69215703..a4307f309d1 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -103,7 +103,7 @@ Response response(loc f, map[str,str] header = ()) = fileResponse(f, mimeTypes[f @synopsis{Utility to quickly serve any rascal value as a json text. This comes in handy for asynchronous HTTP requests from Javascript.} -default Response response(value val, map[str,str] header = ()) = jsonResponse(ok(), header, val); +default Response response(value val, map[str,str] header = (), JSONFormatter[value] formatter = str (value _) { fail; }) = jsonResponse(ok(), header, val, formatter=formatter); @synopsis{Encoding of HTTP status} diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index b24a4d3ce8b..1caabb779b0 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -311,10 +311,39 @@ data CytoStyle ) ; +@synopsis{A combinator language that translates down to strings in JSON} +@description{ +* For field names you can use the names, or the dot notation for array indices and fields of objects: `"labels.0"`, `"name.first"`. +* `and` and `or` can not be nested; this will lead to failure to select anything at all. The or must be outside and the and must be inside. +* `node()` selects all nodes +* `edge()` selects all edges +} data CytoSelector = \node() | \edge() - ; + | \id(str id) + | \and(list[CytoSelector] conjuncts) + | \or(list[CytoSelector] disjuncts) + | \equal(str field, str \value) + | \equal(str field, int limit) + | \greater(str field, int limit) + | \less(str field, int limit) + | \greaterEqual(str field, int limit) + | \lessEqual(str field, int limit) + ; + +@synopsis{Serialize a ((CytoSelector)) to string for client side expression.} +str formatCytoSelector(\node()) = "node"; +str formatCytoSelector(\edge()) = "edge"; +str formatCytoSelector(\id(str i)) = "\"\""; +str formatCytoSelector(and(list[CytoSelector] cjs)) = "<}>"; +str formatCytoSelector(or(list[CytoSelector] cjs)) = ",<}>"[..-1]; +str formatCytoSelector(equal(str field, str val)) = "[ = \"\"]"; +str formatCytoSelector(equal(str field, int lim)) = "[ = ]"; +str formatCytoSelector(greater(str field, int lim)) = "[ \> ]"; +str formatCytoSelector(greaterEqual(str field, int lim)) = "[ \>= ]"; +str formatCytoSelector(lessEqual(str field, int lim)) = "[ \<= ]"; +str formatCytoSelector(less(str field, int lim)) = "[ \< ]"; data CytoLayoutName = grid() @@ -422,7 +451,7 @@ Response (Request) graphServer(Cytoscape ch) { } Response reply(get(/^\/cytoscape/)) { - return response(ch); + return response(ch, formatter=formatCytoSelector); } // returns the main page that also contains the callbacks for retrieving data and configuration From f56300fb22b1e10072bd3f4712245ef128aec3b2 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 20:16:56 +0200 Subject: [PATCH 06/45] start to expand CytoStyleSelector to make use of more styling features --- src/org/rascalmpl/library/vis/Graphs.rsc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 1caabb779b0..c0d3d5f7230 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -230,20 +230,18 @@ data CytoCurveStyle ; data CytoStyleOf - = cytoNodeStyleOf( + = cytoStyleOf( CytoSelector selector = \node(), CytoStyle style = cytoNodeStyle() ) - | cytoEdgeStyleOf( - CytoSelector selector = \edge(), - CytoStyle style = cytoEdgeStyle() - ); + ; -CytoStyleOf cytoNodeStyleOf(CytoStyle style) = cytoNodeStyleOf(selector=\node(), style=style); -CytoStyleOf cytoEdgeStyleOf(CytoStyle style) = cytoEdgeStyleOf(selector=\edge(), style=style); +CytoStyleOf cytoNodeStyleOf(CytoStyle style) = cytoStyleOf(selector=\node(), style=style); +CytoStyleOf cytoEdgeStyleOf(CytoStyle style) = cytoStyleOf(selector=\edge(), style=style); CytoStyle defaultNodeStyle() = cytoNodeStyle( + visibility = "visible", /* hidden, collapse */ width = "label", padding = "10pt", \background-color = "blue", @@ -258,6 +256,7 @@ CytoStyle defaultNodeStyle() CytoStyle defaultEdgeStyle() = cytoEdgeStyle( + visibility = "visible", /* hidden, collapse */ width = 3, \color = "red", \line-color = "black", @@ -278,6 +277,7 @@ data CytoFontWeight data CytoStyle = cytoNodeStyle( + str visibility = "visible", /* hidden, collapse */ str width = "label", str padding = "10pt", str color = "white", @@ -297,6 +297,7 @@ data CytoStyle int \line-height = 1 ) | cytoEdgeStyle( + str visibility = "visible", /* hidden, collapse */ int width = 3, str \line-color = "black", str color = "red", From f48ba151745a86690a88bc0bf2440865d0b33f32 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 20:47:46 +0200 Subject: [PATCH 07/45] new style feature demo with import graph (extend edges are dotted) --- .../library/lang/rascal/vis/ImportGraph.rsc | 17 ++++++++++++- src/org/rascalmpl/library/vis/Graphs.rsc | 25 +++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 0efc06a82a3..1455d207504 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -50,6 +50,21 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. ; + styles = [ + cytoStyleOf( + selector=or([ + and([\node(), id("_")]), // the top node + and([\node(), id("x")]), // the bottom node + and([\edge(), equal("source", "_")]), // edges from the top node + and([\edge(), equal("target", "x")])]), // edges to the bottom node + style=defaultNodeStyle()[visibility="hidden"] // hide it all + ), + cytoStyleOf( + selector=or([ + and([\edge(), equal("label", "E")])]), // extend edges + style=defaultEdgeStyle()[\line-style="dashed"] // are dashed + ) + ]; loc modLinker(str name) { if (loc x <- m.files[name]) return x; @@ -59,7 +74,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { default loc modLinker(value _) = |nothing:///|; - showInteractiveContent(graph(g, \layout=defaultDagreLayout(), nodeLinker=modLinker), title="Rascal Import/Extend Graph"); + showInteractiveContent(graph(g, \layout=defaultDagreLayout(), nodeLinker=modLinker, styles=styles), title="Rascal Import/Extend Graph"); } @synopsis{Container for everything we need to know about the modules in a project to visualize it.} diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index c0d3d5f7230..b767dbfdadb 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -45,8 +45,8 @@ graph(d, \layout=defaultDagreLayout()); graph(d, \layout=defaultDagreLayout(), nodeLabeler=str (loc l) { return l.file; }); ``` } -Content graph(lrel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle()) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle))); +Content graph(lrel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) + = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); @synopsis{A graph plot from a ternary list relation where the middle column is the edge label.} @examples{ @@ -55,8 +55,8 @@ import vis::Graphs; graph([ | x <- [1..100]] + [<100,101,1>]) ``` } -Content graph(lrel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle()) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle))); +Content graph(lrel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) + = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); @synopsis{A graph plot from a binary relation.} @examples{ @@ -65,8 +65,8 @@ import vis::Graphs; graph({ | x <- [1..100]} + {<100,1>}) ``` } -Content graph(rel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle()) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle))); +Content graph(rel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) + = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); @synopsis{A graph plot from a ternary relation where the middle column is the edge label.} @examples{ @@ -75,8 +75,8 @@ import vis::Graphs; graph({ | x <- [1..100]} + {<100,101,1>}) ``` } -Content graph(rel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle()) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle))); +Content graph(rel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) + = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); alias NodeLinker[&T] = loc (&T _id1); loc defaultNodeLinker(/loc l) = l; @@ -91,12 +91,13 @@ alias EdgeLabeler[&T]= str (&T _source, &T _target); str defaultEdgeLabeler(&T _source, &T _target) = ""; -Cytoscape cytoscape(list[CytoData] \data, \CytoLayout \layout=\defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle()) +Cytoscape cytoscape(list[CytoData] \data, \CytoLayout \layout=\defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) = cytoscape( elements=\data, style=[ cytoNodeStyleOf(nodeStyle), - cytoEdgeStyleOf(edgeStyle) + cytoEdgeStyleOf(edgeStyle), + *styles ], \layout=\layout ); @@ -258,6 +259,7 @@ CytoStyle defaultEdgeStyle() = cytoEdgeStyle( visibility = "visible", /* hidden, collapse */ width = 3, + \line-style = "solid", /* dotted, dashed */ \color = "red", \line-color = "black", \target-arrow-color = "black", @@ -300,6 +302,7 @@ data CytoStyle str visibility = "visible", /* hidden, collapse */ int width = 3, str \line-color = "black", + str \line-style = "solid", /* dotted, dashed */ str color = "red", str \target-arrow-color = "black", str \source-arrow-color = "black", @@ -336,7 +339,7 @@ data CytoSelector @synopsis{Serialize a ((CytoSelector)) to string for client side expression.} str formatCytoSelector(\node()) = "node"; str formatCytoSelector(\edge()) = "edge"; -str formatCytoSelector(\id(str i)) = "\"\""; +str formatCytoSelector(\id(str i)) = formatCytoSelector(equal("id", i)); str formatCytoSelector(and(list[CytoSelector] cjs)) = "<}>"; str formatCytoSelector(or(list[CytoSelector] cjs)) = ",<}>"[..-1]; str formatCytoSelector(equal(str field, str val)) = "[ = \"\"]"; From 6fbb9cc15ea30ef5f74a26c9e9deb5e4e276e020 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 21:09:55 +0200 Subject: [PATCH 08/45] transitively closed edges are now only 25 percent visible --- .../rascalmpl/library/lang/rascal/vis/ImportGraph.rsc | 9 +++++++++ src/org/rascalmpl/library/vis/Graphs.rsc | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 1455d207504..5be60af861d 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -50,6 +50,9 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. ; + nonTransitiveEdges = transitiveReduction(g<0,2>); + transitiveEdges = g<0,2> - nonTransitiveEdges; + styles = [ cytoStyleOf( selector=or([ @@ -63,7 +66,13 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { selector=or([ and([\edge(), equal("label", "E")])]), // extend edges style=defaultEdgeStyle()[\line-style="dashed"] // are dashed + ), + *[ cytoStyleOf( + selector=and([\edge(),equal("source", f),equal("target", t)]), + style=defaultEdgeStyle()[opacity="50%"][\line-opacity="0.25"] ) + | <- transitiveEdges + ] ]; loc modLinker(str name) { if (loc x <- m.files[name]) diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index b767dbfdadb..d78df87448d 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -243,6 +243,7 @@ CytoStyleOf cytoEdgeStyleOf(CytoStyle style) = cytoStyleOf(selector=\edge(), sty CytoStyle defaultNodeStyle() = cytoNodeStyle( visibility = "visible", /* hidden, collapse */ + opacity = "1", width = "label", padding = "10pt", \background-color = "blue", @@ -258,6 +259,8 @@ CytoStyle defaultNodeStyle() CytoStyle defaultEdgeStyle() = cytoEdgeStyle( visibility = "visible", /* hidden, collapse */ + opacity = "1", + \line-opacity = "1", width = 3, \line-style = "solid", /* dotted, dashed */ \color = "red", @@ -280,10 +283,11 @@ data CytoFontWeight data CytoStyle = cytoNodeStyle( str visibility = "visible", /* hidden, collapse */ + str opacity = "1", str width = "label", str padding = "10pt", str color = "white", - str \text-opacity = "100%", + str \text-opacity = "1", str \font-family = "", str \font-size = "12pt", str \font-style = "", @@ -300,6 +304,8 @@ data CytoStyle ) | cytoEdgeStyle( str visibility = "visible", /* hidden, collapse */ + str opacity = "1", + str \line-opacity = "1", int width = 3, str \line-color = "black", str \line-style = "solid", /* dotted, dashed */ From a7ba959eada49935518ecd2c88fa42cbd94951a5 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 21:24:12 +0200 Subject: [PATCH 09/45] avoid edge crossings by sorting edges --- .../library/lang/rascal/vis/ImportGraph.rsc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 5be60af861d..933dbdf9078 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -26,6 +26,7 @@ import util::FileSystem; import util::IDEServices; import IO; import analysis::graphs::Graph; +import Set; @synopsis{If `projectName` is an open project in the current IDE, the visualize its import/extend graph.} void importGraph(str projectName, bool hideExternals=true) { @@ -44,14 +45,15 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { m = getProjectModel(pcfg.srcs); // let's start with a simple graph and elaborate on details in later versions - g = { | <- m.imports, hideExternals ==> to notin m.external} - + { | <- m.extends, hideExternals ==> to notin m.external} + g = { | <- sort(m.imports), hideExternals ==> to notin m.external} + + { | <- sort(m.extends), hideExternals ==> to notin m.external} + { <"_", "_", to> | to <- top(m.imports + m.extends) } // pull up the top modules + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. ; - nonTransitiveEdges = transitiveReduction(g<0,2>); - transitiveEdges = g<0,2> - nonTransitiveEdges; + nonTransitiveEdges = transitiveReduction(m.imports + m.extends); + cyclicNodes = { x | <- (m.imports + m.extends)+}; + transitiveEdges = { | <- (m.imports + m.extends), notin nonTransitiveEdges, x notin cyclicNodes, y notin cyclicNodes}; styles = [ cytoStyleOf( @@ -69,7 +71,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { ), *[ cytoStyleOf( selector=and([\edge(),equal("source", f),equal("target", t)]), - style=defaultEdgeStyle()[opacity="50%"][\line-opacity="0.25"] + style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] ) | <- transitiveEdges ] From 70ef51c4e09a77da31aa62e3ed32355e72a8e073 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 21:31:24 +0200 Subject: [PATCH 10/45] details --- src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 933dbdf9078..f92011d20a5 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -70,8 +70,8 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { style=defaultEdgeStyle()[\line-style="dashed"] // are dashed ), *[ cytoStyleOf( - selector=and([\edge(),equal("source", f),equal("target", t)]), - style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] + selector=and([\edge(),equal("source", f),equal("target", t)]), // any transitive edge + style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] // will be made 25% opaque ) | <- transitiveEdges ] From f9ced768d83134151781e94a4475ec4c425e9f16 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 17 Jun 2024 22:14:13 +0200 Subject: [PATCH 11/45] bottom node stretching made layout worse --- .../library/lang/rascal/vis/ImportGraph.rsc | 11 +++++------ src/org/rascalmpl/library/vis/Graphs.rsc | 12 ++++++++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index f92011d20a5..faa469425ce 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -48,20 +48,18 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { g = { | <- sort(m.imports), hideExternals ==> to notin m.external} + { | <- sort(m.extends), hideExternals ==> to notin m.external} + { <"_", "_", to> | to <- top(m.imports + m.extends) } // pull up the top modules - + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. ; nonTransitiveEdges = transitiveReduction(m.imports + m.extends); cyclicNodes = { x | <- (m.imports + m.extends)+}; - transitiveEdges = { | <- (m.imports + m.extends), notin nonTransitiveEdges, x notin cyclicNodes, y notin cyclicNodes}; + transitiveEdges = { | <- (m.imports + m.extends - nonTransitiveEdges), x notin cyclicNodes, y notin cyclicNodes}; styles = [ cytoStyleOf( selector=or([ and([\node(), id("_")]), // the top node - and([\node(), id("x")]), // the bottom node - and([\edge(), equal("source", "_")]), // edges from the top node - and([\edge(), equal("target", "x")])]), // edges to the bottom node + and([\edge(), equal("source", "_")]) // edges from the top node + ]), style=defaultNodeStyle()[visibility="hidden"] // hide it all ), cytoStyleOf( @@ -76,6 +74,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { | <- transitiveEdges ] ]; + loc modLinker(str name) { if (loc x <- m.files[name]) return x; @@ -85,7 +84,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { default loc modLinker(value _) = |nothing:///|; - showInteractiveContent(graph(g, \layout=defaultDagreLayout(), nodeLinker=modLinker, styles=styles), title="Rascal Import/Extend Graph"); + showInteractiveContent(graph(g, \layout=defaultDagreLayout()[ranker=\tight-tree()], nodeLinker=modLinker, styles=styles), title="Rascal Import/Extend Graph"); } @synopsis{Container for everything we need to know about the modules in a project to visualize it.} diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index d78df87448d..ff72af6fe15 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -396,10 +396,17 @@ data CytoLayout(CytoLayoutName name = dagre(), bool animate=false) ) | dagreLayout( CytoLayoutName name = dagre(), - num spacingFactor = .1 + num spacingFactor = .1, + DagreRanker ranker = \network-simplex() // network-simples tight-tree, or longest-path ) ; +data DagreRanker + = \network-simplex() + | \tight-tree() + | \longest-path() + ; + CytoLayout defaultCoseLayout() = coseLayout( name=cose(), @@ -441,7 +448,8 @@ CytoLayout defaultDagreLayout(num spacingFactor=1) = dagreLayout( name=CytoLayoutName::dagre(), animate=false, - spacingFactor=spacingFactor + spacingFactor=spacingFactor, + ranker=\network-simplex() ); From d79c215c64ff9430cf5c10ab0181a02c906809ec Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 10:58:45 +0200 Subject: [PATCH 12/45] refactored vis::Graph to keep all optional config functions into one constructor --- src/org/rascalmpl/library/lang/json/IO.java | 1 - .../library/lang/rascal/vis/ImportGraph.rsc | 29 +++-- src/org/rascalmpl/library/vis/Graphs.rsc | 113 ++++++++++++------ 3 files changed, 99 insertions(+), 44 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 76b7b4fad12..6db83ff7032 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -36,7 +36,6 @@ import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; -import io.usethesource.vallang.ISet; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; import io.usethesource.vallang.IValue; diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index faa469425ce..38908f8c586 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -45,17 +45,23 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { m = getProjectModel(pcfg.srcs); // let's start with a simple graph and elaborate on details in later versions - g = { | <- sort(m.imports), hideExternals ==> to notin m.external} - + { | <- sort(m.extends), hideExternals ==> to notin m.external} - + { <"_", "_", to> | to <- top(m.imports + m.extends) } // pull up the top modules + g = { | <- sort(m.imports), hideExternals ==> to notin m.external} + + { | <- sort(m.extends), hideExternals ==> to notin m.external} + + { <"_" , to> | to <- top(m.imports + m.extends) } // pull up the top modules ; + str nodeClass(str n) = "rascal.project" when n notin m.external; + str nodeClass(str n) = "rascal.external" when n in m.external; + + str edgeClass(str from, str to) = "rascal.import" when in m.imports; + str edgeClass(str from, str to) = "rascal.extend" when in m.extends; + nonTransitiveEdges = transitiveReduction(m.imports + m.extends); - cyclicNodes = { x | <- (m.imports + m.extends)+}; - transitiveEdges = { | <- (m.imports + m.extends - nonTransitiveEdges), x notin cyclicNodes, y notin cyclicNodes}; + cyclicNodes = { x | <- (m.imports + m.extends)+}; + transitiveEdges = { | <- (m.imports + m.extends - nonTransitiveEdges), x notin cyclicNodes, y notin cyclicNodes}; styles = [ - cytoStyleOf( + cytoStyleOf( selector=or([ and([\node(), id("_")]), // the top node and([\edge(), equal("source", "_")]) // edges from the top node @@ -84,7 +90,16 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { default loc modLinker(value _) = |nothing:///|; - showInteractiveContent(graph(g, \layout=defaultDagreLayout()[ranker=\tight-tree()], nodeLinker=modLinker, styles=styles), title="Rascal Import/Extend Graph"); + cfg = cytoGraphConfig( + \layout=dagreLayout(ranker=\tight-tree()), + styles=styles, + title="Rascal Import/Extend Graph", + nodeClassifier=nodeClass, + edgeClassifier=edgeClass, + nodeLinker=modLinker + ); + + showInteractiveContent(graph(g, cfg=cfg), title=cfg.title); } @synopsis{Container for everything we need to know about the modules in a project to visualize it.} diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index ff72af6fe15..2c0d99bf827 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -7,7 +7,7 @@ } @contributor{Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI} @contributor{Tijs van der Storm - storm@cwi.nl - CWI} -@synopsis{Simple data visualization using graphs} +@synopsis{Simple data visualization using graphs; based on cytoscape.js} @description{ This modules provides a simple API to create graph visuals for Rascal (relational) data, based on [Cytoscape.js](https://js.cytoscape.org/). @@ -26,6 +26,41 @@ import util::IDEServices; import Content; import ValueIO; +@synopsis{Optional configuration attributes for graph style and graph layout} +@description{ +These configuration options are used to map input graph data to layout properties +and style properties. + +* title - does what it says +* nodeLinker - makes nodes clickable by providing an editor location +* nodeLabeler - allows simplification or elaboration on node labels beyond their identity string +* nodeClassifier - labels nodes with classes in order to later select them for specific styling +* edgeLabeler - allows simplification or elaboration on edge labels +* layout - defines and configured the graph layout algorithm +* nodeStyle - defines the default style for all nodes +* edgeStyle - defines the default style for all edges +* style - collects specific styles for specific ((CytoSelector)) edge/node selectors using ((CytoStyleOf)) tuples. + +Typically the functions passed into this configuration are closures that capture and use the original +input data to find out about where to link and how to classify. The `&T` parameter reflects the type of +the original input `Graph[&T]`; so that is the type of the nodes. Often this would be `loc` or `str`. +} +data CytoGraphConfig = cytoGraphConfig( + str title="Graph", + + NodeLinker[&T] nodeLinker = defaultNodeLinker, + NodeLabeler[&T] nodeLabeler = defaultNodeLabeler, + NodeClassifier[&T] nodeClassifier = defaultNodeClassifier, + EdgeLabeler[&T] edgeLabeler = defaultEdgeLabeler, + EdgeClassifier[&T] edgeClassifier = defaultEdgeClassifier, + + CytoLayout \layout = defaultCoseLayout(), + + CytoStyle nodeStyle = defaultNodeStyle(), + CytoStyle edgeStyle = defaultEdgeStyle(), + list[CytoStyleOf] styles = [] +); + @synopsis{A graph plot from a binary list relation.} @examples{ ```rascal-shell @@ -45,8 +80,8 @@ graph(d, \layout=defaultDagreLayout()); graph(d, \layout=defaultDagreLayout(), nodeLabeler=str (loc l) { return l.file; }); ``` } -Content graph(lrel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); +Content graph(lrel[&T x, &T y] v, CytoGraphConfig cfg = cytoGraphConfig()) + = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg)))); @synopsis{A graph plot from a ternary list relation where the middle column is the edge label.} @examples{ @@ -55,18 +90,18 @@ import vis::Graphs; graph([ | x <- [1..100]] + [<100,101,1>]) ``` } -Content graph(lrel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); +Content graph(lrel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg), cfg=cfg))); @synopsis{A graph plot from a binary relation.} @examples{ ```rascal-shell import vis::Graphs; graph({ | x <- [1..100]} + {<100,1>}) -``` +``` } -Content graph(rel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler, edgeLabeler=edgeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); +Content graph(rel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg), cfg=cfg))); @synopsis{A graph plot from a ternary relation where the middle column is the edge label.} @examples{ @@ -75,8 +110,8 @@ import vis::Graphs; graph({ | x <- [1..100]} + {<100,101,1>}) ``` } -Content graph(rel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, str title="Graph", CytoLayout \layout=defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) - = content(title, graphServer(cytoscape(graphData(v, nodeLinker=nodeLinker, nodeLabeler=nodeLabeler), \layout=\layout, nodeStyle=nodeStyle, edgeStyle=edgeStyle, styles=styles))); +Content graph(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg), cfg=cfg))); alias NodeLinker[&T] = loc (&T _id1); loc defaultNodeLinker(/loc l) = l; @@ -87,58 +122,64 @@ str defaultNodeLabeler(/str s) = s; str defaultNodeLabeler(loc l) = l.file != "" ? l.file : ""; default str defaultNodeLabeler(&T v) = ""; +alias NodeClassifier[&T] = str (&T _id3); +str defaultNodeClassifier(&T _) = "node"; + +alias EdgeClassifier[&T] = str (&T _from, &T _to); +str defaultEdgeClassifier(&T _, &T _) = "edge"; + alias EdgeLabeler[&T]= str (&T _source, &T _target); str defaultEdgeLabeler(&T _source, &T _target) = ""; -Cytoscape cytoscape(list[CytoData] \data, \CytoLayout \layout=\defaultCoseLayout(), CytoStyle nodeStyle=defaultNodeStyle(), CytoStyle edgeStyle=defaultEdgeStyle(), list[CytoStyleOf] styles=[]) +Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) = cytoscape( elements=\data, style=[ - cytoNodeStyleOf(nodeStyle), - cytoEdgeStyleOf(edgeStyle), - *styles + cytoNodeStyleOf(cfg.nodeStyle), + cytoEdgeStyleOf(cfg.edgeStyle), + *cfg.styles ], - \layout=\layout + \layout=cfg.\layout ); -list[CytoData] graphData(rel[loc x, loc y] v, NodeLinker[loc] nodeLinker=defaultNodeLinker, NodeLabeler[loc] nodeLabeler=defaultNodeLabeler, EdgeLabeler[loc] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=edgeLabeler(from, to))) | <- v] +list[CytoData] graphData(rel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] ; -default list[CytoData] graphData(rel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=edgeLabeler(from, to))) | <- v] +default list[CytoData] graphData(rel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] ; -list[CytoData] graphData(lrel[loc x, &L edge, loc y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + +list[CytoData] graphData(lrel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="")) | <- v] ; -default list[CytoData] graphData(lrel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + +default list[CytoData] graphData(lrel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="")) | <- v] ; -list[CytoData] graphData(lrel[loc x, loc y] v, NodeLinker[loc] nodeLinker=defaultNodeLinker, NodeLabeler[loc] nodeLabeler=defaultNodeLabeler, EdgeLabeler[loc] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=edgeLabeler(from, to))) | <- v] +list[CytoData] graphData(lrel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] ; -default list[CytoData] graphData(lrel[&T x, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=edgeLabeler(from, to))) | <- v] +default list[CytoData] graphData(lrel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] ; -list[CytoData] graphData(rel[loc x, &L edge, loc y] v, NodeLinker[loc] nodeLinker=defaultNodeLinker, NodeLabeler[loc] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + +list[CytoData] graphData(rel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="")) | <- v] ; -default list[CytoData] graphData(rel[&T x, &L edge, &T y] v, NodeLinker[&T] nodeLinker=defaultNodeLinker, NodeLabeler[&T] nodeLabeler=defaultNodeLabeler, EdgeLabeler[&T] edgeLabeler=defaultEdgeLabeler) - = [cytodata(\node("", label=nodeLabeler(e), editor="")) | e <- {*v, *v}] + +default list[CytoData] graphData(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="")) | <- v] ; @@ -181,7 +222,7 @@ data CytoData = cytodata(CytoElement \data); data CytoElement - = \node(str id, str label=id, str editor="|none:///|") + = \node(str id, str label=id, str editor="|none:///|", str class="node") | \edge(str source, str target, str id="-", str label="") ; From fcec7b47bf091dfdb613127f3963bf07a7252932 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 13:50:51 +0200 Subject: [PATCH 13/45] fixes #1979 --- src/org/rascalmpl/interpreter/result/ConstructorFunction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/interpreter/result/ConstructorFunction.java b/src/org/rascalmpl/interpreter/result/ConstructorFunction.java index 54fb43423f4..a20d3fb08e8 100644 --- a/src/org/rascalmpl/interpreter/result/ConstructorFunction.java +++ b/src/org/rascalmpl/interpreter/result/ConstructorFunction.java @@ -157,7 +157,7 @@ public Result computeDefaultKeywordParameter(String label, IConstructor } else { Expression def = getKeywordParameterDefaults().get(kwparam); - IValue res = def.interpret(eval).value; + IValue res = def.interpret(eval).getValue(); if (!res.getType().isSubtypeOf(kwType)) { throw new UnexpectedKeywordArgumentType(kwparam, kwType, res.getType(), ctx.getCurrentAST()); From 620b409f4f0813f7497dacde089d37a795c8fecd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 14:31:55 +0200 Subject: [PATCH 14/45] working with new API for graphs --- .../library/lang/rascal/vis/ImportGraph.rsc | 11 +-- src/org/rascalmpl/library/vis/Graphs.rsc | 80 ++++++++++++------- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 38908f8c586..293635c5fff 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -50,11 +50,8 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { + { <"_" , to> | to <- top(m.imports + m.extends) } // pull up the top modules ; - str nodeClass(str n) = "rascal.project" when n notin m.external; - str nodeClass(str n) = "rascal.external" when n in m.external; - - str edgeClass(str from, str to) = "rascal.import" when in m.imports; - str edgeClass(str from, str to) = "rascal.extend" when in m.extends; + str nodeClass(str n) = n in m.external ? "external" : "project"; + str edgeClass(str from, str to) = in m.imports ? "import" : "extend"; nonTransitiveEdges = transitiveReduction(m.imports + m.extends); cyclicNodes = { x | <- (m.imports + m.extends)+}; @@ -70,7 +67,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { ), cytoStyleOf( selector=or([ - and([\edge(), equal("label", "E")])]), // extend edges + and([\edge(), className("extend")])]), // extend edges style=defaultEdgeStyle()[\line-style="dashed"] // are dashed ), *[ cytoStyleOf( @@ -91,7 +88,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { default loc modLinker(value _) = |nothing:///|; cfg = cytoGraphConfig( - \layout=dagreLayout(ranker=\tight-tree()), + \layout=defaultDagreLayout()[ranker=\network-simplex()], styles=styles, title="Rascal Import/Extend Graph", nodeClassifier=nodeClass, diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 2c0d99bf827..1e502f801c5 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -61,12 +61,46 @@ data CytoGraphConfig = cytoGraphConfig( list[CytoStyleOf] styles = [] ); +@synopsis{A NodeLinker maps node identities to a source location to link to} +alias NodeLinker[&T] = loc (&T _id1); + +@synopsis{The default node linker assumes any loc found in the node identity is a proper link.} +loc defaultNodeLinker(/loc l) = l; +default loc defaultNodeLinker(&T _) = |nothing:///|; + +@synopsis{A NodeLabeler maps node identies to descriptive node labels} +alias NodeLabeler[&T]= str (&T _id2); + +@synopsis{The default node labeler searches for any `str`` in the identity, or otherwise a file name of a `loc`} +str defaultNodeLabeler(/str s) = s; +str defaultNodeLabeler(loc l) = l.file != "" ? l.file : ""; +default str defaultNodeLabeler(&T v) = ""; + +@synopsis{A NodeClassifier maps node identities to classes that are used later to select specific layout and coloring options.} +alias NodeClassifier[&T] = str (&T _id3); + +@synopsis{The default class for all nodes is `"node"``} +str defaultNodeClassifier(&T _) = "node"; + +@synopsis{An EdgeClassifier maps edge identities to classes that are used later to select specific layout and coloring options.} +alias EdgeClassifier[&T] = str (&T _from, &T _to); + +@synopsis{The default class for all edges is `"edge"`} +str defaultEdgeClassifier(&T _, &T _) = "edge"; + +@synopsis{An EdgeLabeler maps edge identies to descriptive edge labels.} +alias EdgeLabeler[&T]= str (&T _source, &T _target); + +@synopsis{The default edge labeler returns the empty label for all edges.} +str defaultEdgeLabeler(&T _source, &T _target) = ""; + + @synopsis{A graph plot from a binary list relation.} @examples{ ```rascal-shell import vis::Graphs; graph([ | x <- [1..100]] + [<100,1>]) -graph([ | x <- [1..100]] + [<100,1>], \layout=\defaultCircleLayout()) +graph([ | x <- [1..100]] + [<100,1>], cfg=cytoGraphConfig(\layout=\defaultCircleLayout())) ``` Providing locations as node identities automatically transforms them to node links: @@ -77,7 +111,7 @@ d = [<|std:///|, e> | e <- |std:///|.ls]; d += [ | <_, e> <- d, isDirectory(e), f <- e.ls]; graph(d, \layout=defaultDagreLayout()); // here we adapt the node labeler to show only the last file name in the path of the location: -graph(d, \layout=defaultDagreLayout(), nodeLabeler=str (loc l) { return l.file; }); +graph(d, \layout=defaultDagreLayout(), cfg=cytoGraphConfig(nodeLabeler=str (loc l) { return l.file; })); ``` } Content graph(lrel[&T x, &T y] v, CytoGraphConfig cfg = cytoGraphConfig()) @@ -113,25 +147,6 @@ graph({ | x <- [1..100]} + {<100,101,1>}) Content graph(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg), cfg=cfg))); -alias NodeLinker[&T] = loc (&T _id1); -loc defaultNodeLinker(/loc l) = l; -default loc defaultNodeLinker(&T _) = |nothing:///|; - -alias NodeLabeler[&T]= str (&T _id2); -str defaultNodeLabeler(/str s) = s; -str defaultNodeLabeler(loc l) = l.file != "" ? l.file : ""; -default str defaultNodeLabeler(&T v) = ""; - -alias NodeClassifier[&T] = str (&T _id3); -str defaultNodeClassifier(&T _) = "node"; - -alias EdgeClassifier[&T] = str (&T _from, &T _to); -str defaultEdgeClassifier(&T _, &T _) = "edge"; - -alias EdgeLabeler[&T]= str (&T _source, &T _target); -str defaultEdgeLabeler(&T _source, &T _target) = ""; - - Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) = cytoscape( elements=\data, @@ -145,42 +160,42 @@ Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) list[CytoData] graphData(rel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; default list[CytoData] graphData(rel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; list[CytoData] graphData(lrel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="")) | <- v] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; default list[CytoData] graphData(lrel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="")) | <- v] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; list[CytoData] graphData(lrel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; default list[CytoData] graphData(lrel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to))) | <- v] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; list[CytoData] graphData(rel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="")) | <- v] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; default list[CytoData] graphData(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="")) | <- v] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; data CytoNodeShape @@ -223,7 +238,7 @@ data CytoData data CytoElement = \node(str id, str label=id, str editor="|none:///|", str class="node") - | \edge(str source, str target, str id="-", str label="") + | \edge(str source, str target, str id="-", str label="", str class="edge") ; data CytoHorizontalAlign @@ -381,14 +396,19 @@ data CytoSelector | \less(str field, int limit) | \greaterEqual(str field, int limit) | \lessEqual(str field, int limit) + | \className(str) ; +@synopsis{Utility to generate class attributes with multiple names consistently.} +CytoSelector classNames(set[str] names) = \className(" <}>"[..-1]); + @synopsis{Serialize a ((CytoSelector)) to string for client side expression.} str formatCytoSelector(\node()) = "node"; str formatCytoSelector(\edge()) = "edge"; str formatCytoSelector(\id(str i)) = formatCytoSelector(equal("id", i)); str formatCytoSelector(and(list[CytoSelector] cjs)) = "<}>"; str formatCytoSelector(or(list[CytoSelector] cjs)) = ",<}>"[..-1]; +str formatCytoSelector(className(str class)) = "."; str formatCytoSelector(equal(str field, str val)) = "[ = \"\"]"; str formatCytoSelector(equal(str field, int lim)) = "[ = ]"; str formatCytoSelector(greater(str field, int lim)) = "[ \> ]"; From b602685036c608683cfa011fc9885d3446f7fb7e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 14:39:56 +0200 Subject: [PATCH 15/45] added doc strings --- src/org/rascalmpl/library/vis/Graphs.rsc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 1e502f801c5..7324c3ae0ba 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -25,6 +25,7 @@ import lang::html::AST; import util::IDEServices; import Content; import ValueIO; +import Set; @synopsis{Optional configuration attributes for graph style and graph layout} @description{ @@ -147,6 +148,11 @@ graph({ | x <- [1..100]} + {<100,101,1>}) Content graph(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = content(cfg.title, graphServer(cytoscape(graphData(v, cfg=cfg), cfg=cfg))); +@synopsis{This core workhorse mixes the graph data with the configuration to obtain visualizable CytoScape.js data-structure.} +@description{ +This data-structure is serialized to JSON and communicated directly to initialize cytoscape.js. +The serialization is done by the generic ((lang::json::IO)) library under the hood of a ((util::Webserver)). +} Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) = cytoscape( elements=\data, @@ -158,41 +164,49 @@ Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) \layout=cfg.\layout ); +@synopsis{Turns a `rel[loc from, loc to]` into a graph} list[CytoData] graphData(rel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `rel[&T from, &T to]` into a graph} default list[CytoData] graphData(rel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `lrel[loc from, &L edge, loc to]` into a graph} list[CytoData] graphData(lrel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `lrel[&T from, &L edge, &T to]` into a graph} default list[CytoData] graphData(lrel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `lrel[loc from, loc to]` into a graph} list[CytoData] graphData(lrel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `lrel[&T from, &T to]` into a graph} default list[CytoData] graphData(lrel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `rel[loc from, &L edge, loc to]` into a graph} list[CytoData] graphData(rel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] ; +@synopsis{Turns any `rel[&T from, &L edge, &T to]` into a graph} default list[CytoData] graphData(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] From 4c35d1203cd32e5cde0a4e3722b61a3fc425bdef Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 22:18:52 +0200 Subject: [PATCH 16/45] fixed the edge classifiers --- .../library/analysis/graphs/Graph.rsc | 3 + .../library/lang/rascal/vis/ImportGraph.rsc | 37 ++++++------ src/org/rascalmpl/library/vis/Graphs.rsc | 60 ++++++++++--------- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/src/org/rascalmpl/library/analysis/graphs/Graph.rsc b/src/org/rascalmpl/library/analysis/graphs/Graph.rsc index dc9301f5c3a..c10bbc51c2e 100644 --- a/src/org/rascalmpl/library/analysis/graphs/Graph.rsc +++ b/src/org/rascalmpl/library/analysis/graphs/Graph.rsc @@ -289,3 +289,6 @@ reduction's worst case complexity is in the same order as transitive closure its * reduces cyclic sub-graphs to "empty" } Graph[&T] transitiveReduction(Graph[&T] g) = g - (g o g+); + +@synopsis{Select the short-cut edges, the ones that transitively close at least two other edges.} +Graph[&T] transitiveEdges(Graph[&T] g) = g o g+; diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 293635c5fff..70475d79c9d 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -50,32 +50,35 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { + { <"_" , to> | to <- top(m.imports + m.extends) } // pull up the top modules ; - str nodeClass(str n) = n in m.external ? "external" : "project"; - str edgeClass(str from, str to) = in m.imports ? "import" : "extend"; - - nonTransitiveEdges = transitiveReduction(m.imports + m.extends); - cyclicNodes = { x | <- (m.imports + m.extends)+}; - transitiveEdges = { | <- (m.imports + m.extends - nonTransitiveEdges), x notin cyclicNodes, y notin cyclicNodes}; + list[str] nodeClass(str n) = [ + *["external" | n in m.external], + *["project" | n notin m.external] + ]; + list[str] edgeClass(str from, str to) = [ + *["extend" | in m.extends], + *["import" | in m.imports], + *["transitive" | in g o g+] + ]; + styles = [ cytoStyleOf( selector=or([ - and([\node(), id("_")]), // the top node - and([\edge(), equal("source", "_")]) // edges from the top node + \node(id("_")), + \edge(equal("source", "_")) ]), - style=defaultNodeStyle()[visibility="hidden"] // hide it all + style=defaultNodeStyle()[visibility="hidden"] ), + cytoStyleOf( - selector=or([ - and([\edge(), className("extend")])]), // extend edges - style=defaultEdgeStyle()[\line-style="dashed"] // are dashed + selector=\edge(className("extend")), + style=defaultEdgeStyle()[\line-style="dashed"] ), - *[ cytoStyleOf( - selector=and([\edge(),equal("source", f),equal("target", t)]), // any transitive edge - style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] // will be made 25% opaque + + cytoStyleOf( + selector=\edge(className("transitive")), + style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] ) - | <- transitiveEdges - ] ]; loc modLinker(str name) { diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 7324c3ae0ba..788629c58f3 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -78,16 +78,16 @@ str defaultNodeLabeler(loc l) = l.file != "" ? l.file : ""; default str defaultNodeLabeler(&T v) = ""; @synopsis{A NodeClassifier maps node identities to classes that are used later to select specific layout and coloring options.} -alias NodeClassifier[&T] = str (&T _id3); +alias NodeClassifier[&T] = list[str] (&T _id3); -@synopsis{The default class for all nodes is `"node"``} -str defaultNodeClassifier(&T _) = "node"; +@synopsis{The default classifier produces no classes} +list[str] defaultNodeClassifier(&T _) = []; @synopsis{An EdgeClassifier maps edge identities to classes that are used later to select specific layout and coloring options.} -alias EdgeClassifier[&T] = str (&T _from, &T _to); +alias EdgeClassifier[&T] = list[str] (&T _from, &T _to); -@synopsis{The default class for all edges is `"edge"`} -str defaultEdgeClassifier(&T _, &T _) = "edge"; +@synopsis{The default edge classifier produces no classes} +list[str] defaultEdgeClassifier(&T _, &T _) = []; @synopsis{An EdgeLabeler maps edge identies to descriptive edge labels.} alias EdgeLabeler[&T]= str (&T _source, &T _target); @@ -166,50 +166,50 @@ Cytoscape cytoscape(list[CytoData] \data, CytoGraphConfig cfg=cytoGraphConfig()) @synopsis{Turns a `rel[loc from, loc to]` into a graph} list[CytoData] graphData(rel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to)), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `rel[&T from, &T to]` into a graph} default list[CytoData] graphData(rel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to)), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `lrel[loc from, &L edge, loc to]` into a graph} list[CytoData] graphData(lrel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=""), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `lrel[&T from, &L edge, &T to]` into a graph} default list[CytoData] graphData(lrel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=""), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `lrel[loc from, loc to]` into a graph} list[CytoData] graphData(lrel[loc x, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to)), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `lrel[&T from, &T to]` into a graph} default list[CytoData] graphData(lrel[&T x, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to), class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=cfg.edgeLabeler(from, to)), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `rel[loc from, &L edge, loc to]` into a graph} list[CytoData] graphData(rel[loc x, &L edge, loc y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=""), classes=cfg.edgeClassifier(from,to)) | <- v] ; @synopsis{Turns any `rel[&T from, &L edge, &T to]` into a graph} default list[CytoData] graphData(rel[&T x, &L edge, &T y] v, CytoGraphConfig cfg=cytoGraphConfig()) - = [cytodata(\node("", label=cfg.nodeLabeler(e), editor="", class=cfg.nodeClassifier(e))) | e <- {*v, *v}] + - [cytodata(\edge("", "", label="", class=cfg.edgeClassifier(from,to))) | <- v] + = [cytodata(\node("", label=cfg.nodeLabeler(e), editor=""), classes=cfg.nodeClassifier(e)) | e <- {*v, *v}] + + [cytodata(\edge("", "", label=""), classes=cfg.edgeClassifier(from,to)) | <- v] ; data CytoNodeShape @@ -248,11 +248,11 @@ data Cytoscape ); data CytoData - = cytodata(CytoElement \data); + = cytodata(CytoElement \data, list[str] classes=[]); data CytoElement - = \node(str id, str label=id, str editor="|none:///|", str class="node") - | \edge(str source, str target, str id="-", str label="", str class="edge") + = \node(str id, str label=id, str editor="|none:///|") + | \edge(str source, str target, str id="-", str label="") ; data CytoHorizontalAlign @@ -384,7 +384,7 @@ data CytoStyle str \source-arrow-color = "black", CytoArrowHeadStyle \target-arrow-shape = CytoArrowHeadStyle::triangle(), CytoArrowHeadStyle \source-arrow-shape = CytoArrowHeadStyle::none(), - CytoCurveStyle \curve-style = CytoCurveStyle::bezier(), + CytoCurveStyle \curve-style = CytoCurveStyle::\unbundled-bezier(), int \source-text-offset = 1, int \target-text-offset = 1, str label = "data(label)" @@ -413,8 +413,14 @@ data CytoSelector | \className(str) ; +@synopsis{Short-hand for a node with a single condition} +CytoSelector \node(CytoSelector condition) = and([\node(), condition]); + +@synopsis{Short-hand for a node with a single condition} +CytoSelector \edge(CytoSelector condition) = and([\edge(), condition]); + @synopsis{Utility to generate class attributes with multiple names consistently.} -CytoSelector classNames(set[str] names) = \className(" <}>"[..-1]); +str more(set[str] names) = " <}>"[..-1]; @synopsis{Serialize a ((CytoSelector)) to string for client side expression.} str formatCytoSelector(\node()) = "node"; From c99359c6fd5e7418c20240dac26b21c9f9879e4e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 22:35:57 +0200 Subject: [PATCH 17/45] highlight cyclic edges --- .../rascalmpl/library/lang/rascal/vis/ImportGraph.rsc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 70475d79c9d..a342f916aa5 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -55,10 +55,13 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { *["project" | n notin m.external] ]; + gClosed = g+; + list[str] edgeClass(str from, str to) = [ *["extend" | in m.extends], *["import" | in m.imports], - *["transitive" | in g o g+] + *["transitive" | in g o gClosed, notin gClosed, notin gClosed], + *["cyclic" | in gClosed] ]; styles = [ @@ -78,6 +81,11 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { cytoStyleOf( selector=\edge(className("transitive")), style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] + ), + + cytoStyleOf( + selector=\edge(className("cyclic")), + style=defaultEdgeStyle()[opacity="1"][\line-opacity="1"][\width=10] ) ]; From f7e268c52b530d86980240017ebfb9e13e84cf9f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 18 Jun 2024 22:58:27 +0200 Subject: [PATCH 18/45] can not style an edge with the defaultNodeStyler (infinite recursio --- .../library/lang/rascal/vis/ImportGraph.rsc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index a342f916aa5..07e0939441c 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -66,11 +66,13 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { styles = [ cytoStyleOf( - selector=or([ - \node(id("_")), - \edge(equal("source", "_")) - ]), - style=defaultNodeStyle()[visibility="hidden"] + selector=\edge(equal("source", "_")), + style=defaultEdgeStyle()[visibility="hidden"] + ), + + cytoStyleOf( + selector=\node(id("_")), + style=defaultNodeStyle()[visibility="hidden"] ), cytoStyleOf( @@ -81,7 +83,8 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { cytoStyleOf( selector=\edge(className("transitive")), style=defaultEdgeStyle()[opacity=".25"][\line-opacity="0.25"] - ), + ) + , cytoStyleOf( selector=\edge(className("cyclic")), From c425f825d0df75d22c55b1acda90c10ad1624f54 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 19 Jun 2024 11:38:39 +0200 Subject: [PATCH 19/45] documented graph styling feature --- src/org/rascalmpl/library/vis/Graphs.rsc | 57 ++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 788629c58f3..c7652926381 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -46,6 +46,63 @@ Typically the functions passed into this configuration are closures that capture input data to find out about where to link and how to classify. The `&T` parameter reflects the type of the original input `Graph[&T]`; so that is the type of the nodes. Often this would be `loc` or `str`. } +@examples{ + +Let's experiment with a number of styling parameters based on the shape of a graph: +```rascal-shell +import vis::Graphs; +// let's play with the geneology of the "Simpsons" +g = { + <"Abraham Simpson", "Homer Simpson">, + <"Mona Simpson", "Homer Simpson">, + <"Homer Simpson", "Bart Simpson">, + <"Homer Simpson", "Lisa Simpson">, + <"Homer Simpson", "Maggie Simpson">, + <"Marge Simpson", "Bart Simpson">, + <"Marge Simpson", "Lisa Simpson">, + <"Marge Simpson", "Maggie Simpson">, + <"Bart Simpson", "Rod Flanders">, + <"Bart Simpson", "Todd Flanders">, + <"Lisa Simpson", "Bart Simpson">, + <"Abraham Simpson", "Patty Bouvier">, + <"Abraham Simpson", "Selma Bouvier">, + <"Mona Simpson", "Patty Bouvier">, + <"Mona Simpson", "Selma Bouvier"> +}; +// visualizing this without styling: +graph(g); +// to style nodes, let's select some special nodes and "classify" them first. We reuse some generic graph analysis tools. +import analysis::graphs::Graph; +list[str] nodeClassifier(str simpson) = [ + *["top" | simpson in top(g)], + *["bottom" | simpson in bottom(g)] +]; +// once classified, we can style each node according to their assigned classes. Nodes can be in more than one class. +styles = [ + cytoStyleOf( + selector=or([\node(className("top")),\node(className("bottom"))]), + style=defaultNodeStyle()[shape=CytoNodeShape::diamond()] + ) +]; +// we pick a sensible layout +lyt = defaultDagreLayout(); +// we wrap the styling information into a configuration wrapper: +cfg = cytoGraphConfig(nodeClassifier=nodeClassifier, styles=styles, \layout=lyt); +// and now we see the effect: +graph(g, cfg=cfg) +// now let's style some edges: +list[str] edgeClassifier(str from, str to) = ["grandparent" | in g o g]; +// add another styling element +styles += [ + cytoStyleOf( + selector=edge(className("grandparent")), + style=defaultEdgeStyle()[\line-style="dashed"] + ) +]; +// and draw again (while adding the grandparent edges too) +graph(g + (g o g), cytoGraphConfig(nodeClassifier=nodeClassifier, edgeClassifier=edgeClassifier, styles=styles, \layout=lyt)) +``` +} data CytoGraphConfig = cytoGraphConfig( str title="Graph", From cd8724f8d96765226d61ac0794ce1499d31a6ae3 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 19 Jun 2024 12:03:57 +0200 Subject: [PATCH 20/45] set tutor to errorsAsWarnings due to bootstrap change in jsonResponse parameters --- pom.xml | 2 +- src/org/rascalmpl/library/Content.rsc | 19 +++++++++++-------- src/org/rascalmpl/library/vis/Graphs.rsc | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index afb406045ce..b130ab325c1 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ false false - false + true ${project.build.outputDirectory} ${project.basedir}/LICENSE |http://github.com/usethesource/rascal/blob/main| diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index a4307f309d1..74cfb32f15f 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -84,27 +84,30 @@ data Response | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }) ; - @synopsis{Utility to quickly render a string as HTML content} Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, content); - @synopsis{Utility to quickly report an HTTP error with a user-defined message} Response response(Status status, str explanation, map[str,str] header = ()) = response(status, "text/plain", header, explanation); - @synopsis{Utility to quickly make a plaintext response.} Response plain(str text) = response(ok(), "text/plain", (), text); - @synopsis{Utility to serve a file from any source location.} Response response(loc f, map[str,str] header = ()) = fileResponse(f, mimeTypes[f.extension]?"text/plain", header); +@synopsis{Utility to quickly serve any rascal value as a json text.} +@benefits{ +This comes in handy for asynchronous HTTP requests from Javascript clients. Rascal Values are +fully transformed to their respective JSON serialized form before being communicated over HTTP. +} +default Response response(value val, map[str,str] header = ()) = jsonResponse(ok(), header, val); -@synopsis{Utility to quickly serve any rascal value as a json text. This comes in handy for -asynchronous HTTP requests from Javascript.} -default Response response(value val, map[str,str] header = (), JSONFormatter[value] formatter = str (value _) { fail; }) = jsonResponse(ok(), header, val, formatter=formatter); - +@synopsis{Utility to quickly serve any rascal value as a json text, formatting data-types on-the-fly using a `formatter` function} +@benefits{ +Fast way of producing JSON strings for embedded DSLs on the Rascal side. +} +Response response(value val, JSONFormatter[value] formatter, map[str,str] header = ()) = jsonResponse(ok(), header, val, formatter=formatter); @synopsis{Encoding of HTTP status} data Status diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index c7652926381..4c93dcbab02 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -607,7 +607,7 @@ Response (Request) graphServer(Cytoscape ch) { } Response reply(get(/^\/cytoscape/)) { - return response(ch, formatter=formatCytoSelector); + return response(ch, formatCytoSelector); } // returns the main page that also contains the callbacks for retrieving data and configuration From 7b66cd28c7d54492392171300d32a3adb1582dbd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 19 Jun 2024 12:45:07 +0200 Subject: [PATCH 21/45] better curves --- src/org/rascalmpl/library/vis/Graphs.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 4c93dcbab02..433a51aa645 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -396,7 +396,7 @@ CytoStyle defaultEdgeStyle() \source-arrow-color = "black", \target-arrow-shape = CytoArrowHeadStyle::triangle(), \source-arrow-shape = CytoArrowHeadStyle::none(), - \curve-style = bezier(), + \curve-style = \unbundled-bezier(), \label = "data(label)" ); From e1599182735dee27ee7283524778677069ce4e5f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 26 Jun 2024 11:59:55 +0200 Subject: [PATCH 22/45] this fixes #1987 but breaks the handling of default keyword parameters. Have to bring that back somehow by expecting NULL in the specific loops for keyword parameters --- src/org/rascalmpl/library/lang/json/IO.java | 18 +++++- src/org/rascalmpl/library/lang/json/IO.rsc | 27 +++++++- .../lang/json/internal/JsonValueReader.java | 61 ++++++++++++++++--- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 6db83ff7032..d4f56a375ba 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -19,6 +19,12 @@ import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.Charset; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; @@ -36,8 +42,10 @@ import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; +import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; +import io.usethesource.vallang.ITuple; import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeStore; @@ -102,7 +110,7 @@ public IValue fromJSON(IValue type, IString src) { } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { + public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -116,6 +124,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) .setCalendarFormat(dateTimeFormat.getValue()) .setParsers(parsers) + .setNulls(unreify(null)) .read(in, start); } catch (IOException e) { @@ -127,6 +136,13 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, } } + private Map unreify(IMap nulls) { + var tr = new TypeReifier(values); + return nulls.stream() + .map(t -> (ITuple) t) + .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + } + public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 740c14f4e2a..c70df892214 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -14,6 +14,8 @@ module lang::json::IO +import util::Maybe; + @javaClass{org.rascalmpl.library.lang.json.IO} @deprecated{ use writeJSON @@ -45,8 +47,31 @@ In general the translation behaves as follows: * If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]} * Go to ((JSONParser)) to find out how to use the optional `parsers` parameter. +* if the parser finds a `null` JSON value, it will lookup in the `nulls` map based on the currently expected type which value to return, or throw an exception otherwise. +First the expected type is used as a literal lookup, and then each value is tested if the current type is a subtype of it. } -java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }); +java &T readJSON( + type[&T] expected, + loc src, + str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", + bool lenient=false, + bool trackOrigins=false, + JSONParser[value] parser = (type[value] _, str _) { throw ""; }, + map [type[value] forType, value nullValue] nulls = defaultJSONNULLValues); + +public map[type[value] forType, value nullValue] defaultJSONNULLValues = ( + #Maybe[value] : nothing(), + #node : "null"(), + #int : -1, + #real : -1.0, + #rat :-1r1, + #value : "null"(), + #str : "", + #list[value] : [], + #set[value] : {}, + #map[value,value] : (), + #loc : |unknown:///| +); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{parses JSON values from a string. diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index d0bd83f90b2..a5ef6b33274 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -66,6 +66,7 @@ public class JsonValueReader { private VarHandle lineHandler; private VarHandle lineStartHandler; private IFunction parsers; + private Map nulls = Collections.emptyMap(); /** * @param vf factory which will be used to construct values @@ -98,6 +99,10 @@ public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLo this(vf, new TypeStore(), monitor, src); } + public JsonValueReader setNulls(Map nulls) { + this.nulls = nulls; + return this; + } /** * Builder method to set the format to use for all date-time values encoded as strings */ @@ -140,7 +145,8 @@ public IValue visitInteger(Type type) throws IOException { case STRING: return vf.integer(in.nextString()); case NULL: - return null; + in.nextNull(); + return inferNullValue(nulls, type); default: throw new IOException("Expected integer but got " + in.peek()); } @@ -158,7 +164,8 @@ public IValue visitReal(Type type) throws IOException { case STRING: return vf.real(in.nextString()); case NULL: - return null; + in.nextNull(); + return inferNullValue(nulls, type); default: throw new IOException("Expected integer but got " + in.peek()); } @@ -168,6 +175,38 @@ public IValue visitReal(Type type) throws IOException { } } + private IValue inferNullValue(Map nulls, Type expected) { + IValue nullValue = nulls.get(expected); + + if (nullValue != null) { + return nullValue; + } + + for (Type superType : nulls.keySet()) { + if (superType.isTop() || superType.isNode()) { + continue; // those are last resorts. + } + + if (expected.isSubtypeOf(superType)) { + return nulls.get(superType); + } + } + + Type node = TypeFactory.getInstance().nodeType(); + Type value = TypeFactory.getInstance().valueType(); + + if (expected.isSubtypeOf(node) && nulls.containsKey(node)) { + return nulls.get(node); + } + + if (expected.isSubtypeOf(value) && nulls.containsKey(value)) { + return nulls.get(value); + } + + /* this will trigger an NPE somewhere */ + return null; + } + @Override public IValue visitExternal(Type type) throws IOException { throw new IOException("External type " + type + "is not implemented yet by the json reader:" + in.getPath()); @@ -176,7 +215,7 @@ public IValue visitExternal(Type type) throws IOException { @Override public IValue visitString(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } return vf.string(in.nextString()); @@ -185,7 +224,7 @@ public IValue visitString(Type type) throws IOException { @Override public IValue visitTuple(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } List l = new ArrayList<>(); @@ -334,7 +373,7 @@ public IValue visitValue(Type type) throws IOException { return vf.string(in.nextName()); case NULL: in.nextNull(); - return null; + return inferNullValue(nulls, type); default: throw new IOException("Did not expect end of Json value here, while looking for " + type + " + at " + in.getPath()); } @@ -361,7 +400,7 @@ else if (val.contains("://")) { public IValue visitRational(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } switch (in.peek()) { @@ -400,7 +439,7 @@ public IValue visitRational(Type type) throws IOException { @Override public IValue visitMap(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } IMapWriter w = vf.mapWriter(); @@ -444,7 +483,7 @@ public IValue visitAlias(Type type) throws IOException { @Override public IValue visitBool(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } return vf.bool(in.nextBoolean()); } @@ -661,7 +700,7 @@ public IValue visitDateTime(Type type) throws IOException { @Override public IValue visitList(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } IListWriter w = vf.listWriter(); @@ -677,7 +716,7 @@ public IValue visitList(Type type) throws IOException { public IValue visitSet(Type type) throws IOException { if (isNull()) { - return null; + return inferNullValue(nulls, type); } ISetWriter w = vf.setWriter(); @@ -706,4 +745,6 @@ private boolean isNull() throws IOException { return res; } + + } \ No newline at end of file From 596755a3f1038e5e7e8f626fce6b3a33735217d6 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 26 Jun 2024 13:24:35 +0200 Subject: [PATCH 23/45] fixed default values for keyword parameters in JSON parser --- .../lang/json/internal/JsonValueReader.java | 54 +++++++------------ 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index a5ef6b33274..f8382b9fccd 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -176,35 +176,14 @@ public IValue visitReal(Type type) throws IOException { } private IValue inferNullValue(Map nulls, Type expected) { - IValue nullValue = nulls.get(expected); - - if (nullValue != null) { - return nullValue; - } - - for (Type superType : nulls.keySet()) { - if (superType.isTop() || superType.isNode()) { - continue; // those are last resorts. - } - - if (expected.isSubtypeOf(superType)) { - return nulls.get(superType); - } - } - - Type node = TypeFactory.getInstance().nodeType(); - Type value = TypeFactory.getInstance().valueType(); - - if (expected.isSubtypeOf(node) && nulls.containsKey(node)) { - return nulls.get(node); - } - - if (expected.isSubtypeOf(value) && nulls.containsKey(value)) { - return nulls.get(value); - } - - /* this will trigger an NPE somewhere */ - return null; + return nulls.keySet().stream() + .sorted((x,y) -> x.compareTo(y)) // smaller types are matched first + .filter(superType -> expected.isSubtypeOf(superType)) // remove any type that does not fit + .findFirst() // give the most specific match + .map(t -> nulls.get(t)) // lookup the corresponding null value + .orElse(null); // or muddle on and throw NPE elsewhere + + // The NPE triggering "elsewhere" should help with fault localization. } @Override @@ -536,6 +515,10 @@ private int getCol() { @Override public IValue visitAbstractData(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); + } + if (in.peek() == JsonToken.STRING) { var stringInput = in.nextString(); @@ -573,7 +556,7 @@ public IValue visitAbstractData(Type type) throws IOException { } } - assert in.peek() == JsonToken.BEGIN_OBJECT; + assert in.peek() == JsonToken.BEGIN_OBJECT || in.peek() == JsonToken.NULL; Set alternatives = store.lookupAlternatives(type); if (alternatives.size() > 1) { @@ -605,10 +588,13 @@ public IValue visitAbstractData(Type type) throws IOException { } } else if (cons.hasKeywordField(label, store)) { - IValue val = read(in, store.getKeywordParameterType(cons, label)); - if (val != null) { - // if the value is null we'd use the default value of the defined field in the constructor - kwParams.put(label, val); + if (!isNull()) { // lookahead for null to give default parameters the preference. + IValue val = read(in, store.getKeywordParameterType(cons, label)); + // null can still happen if the nulls map doesn't have a default + if (val != null) { + // if the value is null we'd use the default value of the defined field in the constructor + kwParams.put(label, val); + } } } else { // its a normal arg, pass its label to the child From 00a391cf580578bdb89a056f85123565aa1ab55b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 10:16:12 +0200 Subject: [PATCH 24/45] refactored and added explicit support for Maybe --- src/org/rascalmpl/library/lang/json/IO.java | 8 +- src/org/rascalmpl/library/lang/json/IO.rsc | 10 +- .../lang/json/internal/JsonValueReader.java | 132 +++++++++++++----- 3 files changed, 108 insertions(+), 42 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index d4f56a375ba..cd960c9d7ab 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -20,12 +20,7 @@ import java.io.StringWriter; import java.nio.charset.Charset; import java.util.Map; -import java.util.Map.Entry; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.library.lang.json.internal.IValueAdapter; @@ -143,7 +138,7 @@ private Map unreify(IMap nulls) { .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); } - public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers) { + public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -153,6 +148,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null) .setCalendarFormat(dateTimeFormat.getValue()) .setParsers(parsers) + .setNulls(unreify(nulls)) .read(in, start); } catch (IOException e) { diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index c70df892214..e642ef7231a 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -76,7 +76,15 @@ public map[type[value] forType, value nullValue] defaultJSONNULLValues = ( @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{parses JSON values from a string. In general the translation behaves as the same as for ((readJSON)).} -java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }); +java &T parseJSON( + type[&T] expected, + str src, + str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", + bool lenient=false, + bool trackOrigins=false, + JSONParser[value] parser = (type[value] _, str _) { throw ""; }, + map[type[value] forType, value nullValue] null = defaultJSONNULLValues +); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{writes `val` to the location `target`} diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index f8382b9fccd..1d300675c64 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -67,6 +67,11 @@ public class JsonValueReader { private VarHandle lineStartHandler; private IFunction parsers; private Map nulls = Collections.emptyMap(); + private final Type ParameterT = TF.parameterType("A"); + private final Type Maybe; + private final Type Maybe_nothing; + private final Type Maybe_just; + /** * @param vf factory which will be used to construct values @@ -77,6 +82,12 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m this.store = store; this.monitor = monitor; this.src = src; + + // WARNING: this clones the definitions of util::Maybe until we can reuse compiler-generated code + this.Maybe = TF.abstractDataType(store, "Maybe", ParameterT); + this.Maybe_nothing = TF.constructor(store, Maybe, "nothing"); + this.Maybe_just = TF.constructor(store, Maybe, "just", ParameterT, "val"); + setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); if (src != null) { @@ -94,6 +105,14 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m } } } + + private IConstructor justCons(IValue val) { + return vf.constructor(Maybe_just, val); + } + + private final IConstructor nothingCons() { + return vf.constructor(Maybe_nothing); + } public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { this(vf, new TypeStore(), monitor, src); @@ -512,52 +531,71 @@ private int getCol() { } } - - @Override - public IValue visitAbstractData(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); + /** + * Expecting an ADT we found NULL on the lookahead. + * This is either a Maybe or we can use the map + * of null values. + */ + private IValue visitNullAsAbstractData(Type type) { + if (type.isSubtypeOf(Maybe)) { + return nothingCons(); } - if (in.peek() == JsonToken.STRING) { - var stringInput = in.nextString(); + return inferNullValue(nulls, type); + } - // might be a parsable string. let's see. - if (parsers != null) { - var reified = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); - - try { - return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); - } - catch (Throw t) { - Type excType = t.getException().getType(); + /** + * Expecting an ADT we found a string value instead. + * Now we can (try to) apply the parsers that were passed in. + * If that does not fly, we can interpret strings as nullary ADT + * constructors. + */ + private IValue visitStringAsAbstractData(Type type) throws IOException { + var stringInput = in.nextString(); + + // might be a parsable string. let's see. + if (parsers != null) { + var reified = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); + + try { + return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); + } + catch (Throw t) { + Type excType = t.getException().getType(); - if (excType.isAbstractData() && ((IConstructor) t.getException()).getConstructorType().getName().equals("ParseError")) { - throw new IOException(t); // an actual parse error is meaningful to report - } - // otherwise we fall through to enum recognition + if (excType.isAbstractData() && ((IConstructor) t.getException()).getConstructorType().getName().equals("ParseError")) { + throw new IOException(t); // an actual parse error is meaningful to report } + // otherwise we fall through to enum recognition } + } - // enum! - Set enumCons = store.lookupConstructor(type, stringInput); + // enum! + Set enumCons = store.lookupConstructor(type, stringInput); - for (Type candidate : enumCons) { - if (candidate.getArity() == 0) { - return vf.constructor(candidate); - } - } - - if (parsers != null) { - throw new IOException("parser failed to recognize \"" + stringInput + "\" and no nullary constructor found for " + type + "either"); - } - else { - throw new IOException("no nullary constructor found for " + type); + for (Type candidate : enumCons) { + if (candidate.getArity() == 0) { + return vf.constructor(candidate); } } + + if (parsers != null) { + throw new IOException("parser failed to recognize \"" + stringInput + "\" and no nullary constructor found for " + type + "either"); + } + else { + throw new IOException("no nullary constructor found for " + type + ", that matches " + stringInput); + } + } - assert in.peek() == JsonToken.BEGIN_OBJECT || in.peek() == JsonToken.NULL; - + /** + * This is the main workhorse. Every object is mapped one-to-one to an ADT constructor + * instance. The field names (keyword parameters and positional) are mapped to field + * names of the object. The name of the constructor is _not_ consequential. + * @param type + * @return + * @throws IOException + */ + private IValue visitObjectAsAbstractData(Type type) throws IOException { Set alternatives = store.lookupAlternatives(type); if (alternatives.size() > 1) { monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); @@ -620,6 +658,30 @@ else if (cons.hasKeywordField(label, store)) { return vf.constructor(cons, args, kwParams); } + + @Override + public IValue visitAbstractData(Type type) throws IOException { + if (type.isSubtypeOf(Maybe)) { + if (in.peek() == JsonToken.NULL) { + return nothingCons(); + } + else { + // dive into the wrapped type, and wrap the result. Could be a str, int, or anything. + return justCons(type.getTypeParameters().getFieldType(0).accept(this)); + } + } + + switch (in.peek()) { + case NULL: + return visitNullAsAbstractData(type); + case STRING: + return visitStringAsAbstractData(type); + case BEGIN_OBJECT: + return visitObjectAsAbstractData(type); + default: + throw new IOException("Expected ADT:" + type + ", but found " + in.peek().toString()); + } + } @Override public IValue visitConstructor(Type type) throws IOException { From 0a6cf52d509f5c251882cfa9af7c00464f96e4b8 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 10:26:22 +0200 Subject: [PATCH 25/45] factored out UtilMaybe for future maintainability --- .../lang/json/internal/JsonValueReader.java | 28 ++--------- src/org/rascalmpl/values/maybe/UtilMaybe.java | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 src/org/rascalmpl/values/maybe/UtilMaybe.java diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 1d300675c64..5f1eda57493 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -33,6 +33,7 @@ import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; +import org.rascalmpl.values.maybe.UtilMaybe; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; @@ -67,10 +68,6 @@ public class JsonValueReader { private VarHandle lineStartHandler; private IFunction parsers; private Map nulls = Collections.emptyMap(); - private final Type ParameterT = TF.parameterType("A"); - private final Type Maybe; - private final Type Maybe_nothing; - private final Type Maybe_just; /** @@ -83,11 +80,6 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m this.monitor = monitor; this.src = src; - // WARNING: this clones the definitions of util::Maybe until we can reuse compiler-generated code - this.Maybe = TF.abstractDataType(store, "Maybe", ParameterT); - this.Maybe_nothing = TF.constructor(store, Maybe, "nothing"); - this.Maybe_just = TF.constructor(store, Maybe, "just", ParameterT, "val"); - setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); if (src != null) { @@ -105,14 +97,6 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m } } } - - private IConstructor justCons(IValue val) { - return vf.constructor(Maybe_just, val); - } - - private final IConstructor nothingCons() { - return vf.constructor(Maybe_nothing); - } public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { this(vf, new TypeStore(), monitor, src); @@ -537,10 +521,6 @@ private int getCol() { * of null values. */ private IValue visitNullAsAbstractData(Type type) { - if (type.isSubtypeOf(Maybe)) { - return nothingCons(); - } - return inferNullValue(nulls, type); } @@ -661,13 +641,13 @@ else if (cons.hasKeywordField(label, store)) { @Override public IValue visitAbstractData(Type type) throws IOException { - if (type.isSubtypeOf(Maybe)) { + if (UtilMaybe.isMaybe(type)) { if (in.peek() == JsonToken.NULL) { - return nothingCons(); + return UtilMaybe.nothing(); } else { // dive into the wrapped type, and wrap the result. Could be a str, int, or anything. - return justCons(type.getTypeParameters().getFieldType(0).accept(this)); + return UtilMaybe.just(type.getTypeParameters().getFieldType(0).accept(this)); } } diff --git a/src/org/rascalmpl/values/maybe/UtilMaybe.java b/src/org/rascalmpl/values/maybe/UtilMaybe.java new file mode 100644 index 00000000000..d44b26f4937 --- /dev/null +++ b/src/org/rascalmpl/values/maybe/UtilMaybe.java @@ -0,0 +1,46 @@ +package org.rascalmpl.values.maybe; + +import org.rascalmpl.values.IRascalValueFactory; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeFactory; +import io.usethesource.vallang.type.TypeStore; + +/** + * Static factory for Maybe[&A] instances. + * WARNING: this clones the definitions of util::Maybe until we can reuse compiler-generated code + */ +public class UtilMaybe { + private static final TypeStore store = new TypeStore(); + private static final TypeFactory TF = TypeFactory.getInstance(); + private static final Type ParameterT = TF.parameterType("A"); + public static final Type Maybe; + private static final Type Maybe_nothing; + private static final Type Maybe_just; + + static { + Maybe = TF.abstractDataType(store, "Maybe", ParameterT); + Maybe_nothing = TF.constructor(store, Maybe, "nothing"); + Maybe_just = TF.constructor(store, Maybe, "just", ParameterT, "val"); + } + + public static boolean isMaybe(Type t) { + return t.isSubtypeOf(Maybe); + } + + /** + * create `just(val)` of type `Maybe[typeOf(val)]` + */ + public static IConstructor just(IValue val) { + return IRascalValueFactory.getInstance().constructor(Maybe_just, val); + } + + /** + * Create `nothing()` of type `Maybe[void]` + */ + public static IConstructor nothing() { + return IRascalValueFactory.getInstance().constructor(Maybe_nothing); + } +} From 73086f27a98668c80180e38855143b6744744eff Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 10:36:31 +0200 Subject: [PATCH 26/45] fixed bug --- src/org/rascalmpl/library/lang/json/IO.rsc | 2 +- .../rascalmpl/library/lang/json/internal/JsonValueReader.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index e642ef7231a..eef9a6c1027 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -83,7 +83,7 @@ java &T parseJSON( bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }, - map[type[value] forType, value nullValue] null = defaultJSONNULLValues + map[type[value] forType, value nullValue] nulls = defaultJSONNULLValues ); @javaClass{org.rascalmpl.library.lang.json.IO} diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 5f1eda57493..c3c38087959 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -643,6 +643,7 @@ else if (cons.hasKeywordField(label, store)) { public IValue visitAbstractData(Type type) throws IOException { if (UtilMaybe.isMaybe(type)) { if (in.peek() == JsonToken.NULL) { + in.nextNull(); return UtilMaybe.nothing(); } else { From cdab1db283b0f9e30de3a374256c4d8af962ae22 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 10:49:44 +0200 Subject: [PATCH 27/45] more checking on the provided null values --- .../lang/json/internal/JsonValueReader.java | 5 ++++ .../tests/library/lang/json/JSONIOTests.rsc | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index c3c38087959..0794ede2e62 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -184,6 +184,7 @@ private IValue inferNullValue(Map nulls, Type expected) { .filter(superType -> expected.isSubtypeOf(superType)) // remove any type that does not fit .findFirst() // give the most specific match .map(t -> nulls.get(t)) // lookup the corresponding null value + .filter(r -> r.getType().isSubtypeOf(expected)) // the value in the table still has to fit the currently expected type .orElse(null); // or muddle on and throw NPE elsewhere // The NPE triggering "elsewhere" should help with fault localization. @@ -671,6 +672,10 @@ public IValue visitConstructor(Type type) throws IOException { @Override public IValue visitNode(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); + } + in.beginObject(); int startPos = getPos(); int startLine = getLine(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index ef8e837ee0f..21e5c9acc61 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -3,6 +3,7 @@ module lang::rascal::tests::library::lang::json::JSONIOTests import String; import lang::json::IO; import util::UUID; +import util::Maybe; import IO; loc targetFile = |memory://test-tmp/test-<"">.json|; @@ -69,4 +70,29 @@ test bool originTracking() { } return true; +} + +data Cons = cons(str bla = "null"); + +test bool dealWithNull() { + // use the default nulls map + assert parseJSON(#map[str,value], "{\"bla\": null}") == ("bla":"null"()); + + // using our own nulls map + assert parseJSON(#map[str,value], "{\"bla\": null}", nulls=(#value:-1)) == ("bla":-1); + + // conflicting entries in the nulls maps: more specific goes first: + assert parseJSON(#map[str,node], "{\"bla\": null}", nulls=(#value:-1, #node:"null"())) == ("bla":"null"()); + + // the builtin Maybe interpreter with null + assert parseJSON(#map[str,Maybe[str]], "{\"bla\": null}") == ("bla":nothing()); + + // the builtin Maybe interpreter with non-null + assert parseJSON(#map[str,Maybe[str]], "{\"bla\": \"foo\"}") == ("bla":just("foo")); + + // keyword parameters and null + assert parseJSON(#Cons, "{\"bla\": \"foo\"}") == cons(bla="foo"); + assert parseJSON(#Cons, "{\"bla\": null}") == cons(); + + return true; } \ No newline at end of file From 7cff3fdeb6a24f52d367a60fbdfa4d8667dd5e29 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 10:56:20 +0200 Subject: [PATCH 28/45] writer can deal with Maybe too --- .../library/lang/json/internal/JsonValueWriter.java | 10 ++++++++++ src/org/rascalmpl/values/maybe/UtilMaybe.java | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index d72bf3790ed..72d7886a88d 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -19,6 +19,7 @@ import org.rascalmpl.exceptions.Throw; import org.rascalmpl.values.functions.IFunction; +import org.rascalmpl.values.maybe.UtilMaybe; import com.google.gson.stream.JsonWriter; @@ -236,6 +237,15 @@ public Void visitNode(INode o) throws IOException { @Override public Void visitConstructor(IConstructor o) throws IOException { + if (UtilMaybe.isMaybe(o.getType())) { + if (UtilMaybe.isNothing(o)) { + out.nullValue(); + } + else { + o.get(0).accept(this); + } + } + if (formatters != null) { try { var formatted = formatters.call(o); diff --git a/src/org/rascalmpl/values/maybe/UtilMaybe.java b/src/org/rascalmpl/values/maybe/UtilMaybe.java index d44b26f4937..4dc79c7604a 100644 --- a/src/org/rascalmpl/values/maybe/UtilMaybe.java +++ b/src/org/rascalmpl/values/maybe/UtilMaybe.java @@ -30,6 +30,14 @@ public static boolean isMaybe(Type t) { return t.isSubtypeOf(Maybe); } + public static boolean isNothing(IConstructor v) { + return v.getConstructorType().getName().equals("nothing"); + } + + public static boolean isJust(IConstructor v) { + return !isNothing(v); + } + /** * create `just(val)` of type `Maybe[typeOf(val)]` */ From 9768cb83d109bb5d34f364294962eb5b188a30ac Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 11:54:33 +0200 Subject: [PATCH 29/45] radically improved error reporting of JSON parser --- .../exceptions/RuntimeExceptionFactory.java | 7 + src/org/rascalmpl/library/lang/json/IO.java | 2 +- src/org/rascalmpl/library/lang/json/IO.rsc | 8 + .../lang/json/internal/JsonValueReader.java | 1264 +++++++++-------- 4 files changed, 660 insertions(+), 621 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 50c37b54b61..2e54b541f5d 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -649,6 +649,13 @@ public static Throw noSuchKey(IValue v, AbstractAST ast, StackTrace trace) { public static Throw parseError(ISourceLocation loc) { return new Throw(VF.constructor(ParseError, loc)); } + + public static Throw jsonParseError(ISourceLocation loc, String cause, String path) { + return new Throw(VF.constructor(ParseError, loc) + .asWithKeywordParameters().setParameter("reason", VF.string(cause)) + .asWithKeywordParameters().setParameter("path", VF.string(path))); + } + public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index cd960c9d7ab..cc8c4929888 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -119,7 +119,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) .setCalendarFormat(dateTimeFormat.getValue()) .setParsers(parsers) - .setNulls(unreify(null)) + .setNulls(unreify(nulls)) .read(in, start); } catch (IOException e) { diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index eef9a6c1027..ac97e2762e6 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -16,6 +16,14 @@ module lang::json::IO import util::Maybe; +@synopsis{JSON parse errors have more information than general parse errors} +@description{ +* `location` is the place where the parsing got stuck (going from left to right). +* `cause` is a factual diagnosis of what was expected at that position, versus what was found. +* `path` is a path query string into the JSON value from the root down to the leaf where the error was detected. +} +data RuntimeException = ParseError(loc location, str cause="", str path=""); + @javaClass{org.rascalmpl.library.lang.json.IO} @deprecated{ use writeJSON diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 0794ede2e62..65e6c0e3c85 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -12,6 +12,7 @@ */ package org.rascalmpl.library.lang.json.internal; +import java.io.EOFException; import java.io.IOException; import java.io.StringReader; import java.lang.invoke.MethodHandles; @@ -27,7 +28,9 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang.ObjectUtils.Null; import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; import org.rascalmpl.types.ReifiedType; import org.rascalmpl.uri.URIUtil; @@ -48,8 +51,11 @@ import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; +import com.google.gson.stream.MalformedJsonException; /** * This class streams a JSON stream directly to an IValue representation and validates the content @@ -57,728 +63,746 @@ */ public class JsonValueReader { - private static final TypeFactory TF = TypeFactory.getInstance(); - private final TypeStore store; - private final IRascalValueFactory vf; - private ThreadLocal format; - private final IRascalMonitor monitor; - private ISourceLocation src; - private VarHandle posHandler; - private VarHandle lineHandler; - private VarHandle lineStartHandler; - private IFunction parsers; - private Map nulls = Collections.emptyMap(); - - - /** - * @param vf factory which will be used to construct values - * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields - */ - public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { - this.vf = vf; - this.store = store; - this.monitor = monitor; - this.src = src; + private final class ExpectedTypeDispatcher implements ITypeVisitor { + private final JsonReader in; - setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + private ExpectedTypeDispatcher(JsonReader in) { + this.in = in; + } - if (src != null) { + @Override + public IValue visitInteger(Type type) throws IOException { try { - var lookup = MethodHandles.lookup(); - var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); - this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); - this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); - this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); - } - catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { - // we disable the origin tracking if we can not get to the fields - src = null; - monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + switch (in.peek()) { + case NUMBER: + return vf.integer(in.nextLong()); + case STRING: + return vf.integer(in.nextString()); + case NULL: + in.nextNull(); + return inferNullValue(nulls, type); + default: + throw parseErrorHere("Expected integer but got " + in.peek()); + } + } + catch (NumberFormatException e) { + throw parseErrorHere("Expected integer but got " + e.getMessage()); } } - } - - public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { - this(vf, new TypeStore(), monitor, src); - } - - public JsonValueReader setNulls(Map nulls) { - this.nulls = nulls; - return this; - } - /** - * Builder method to set the format to use for all date-time values encoded as strings - */ - public JsonValueReader setCalendarFormat(String format) { - // SimpleDateFormat is not thread safe, so here we make sure - // we can use objects of this reader in different threads at the same time - this.format = new ThreadLocal() { - protected SimpleDateFormat initialValue() { - return new SimpleDateFormat(format); + + public IValue visitReal(Type type) throws IOException { + try { + switch (in.peek()) { + case NUMBER: + return vf.real(in.nextDouble()); + case STRING: + return vf.real(in.nextString()); + case NULL: + in.nextNull(); + return inferNullValue(nulls, type); + default: + throw parseErrorHere("Expected integer but got " + in.peek()); + } } - }; - return this; - } + catch (NumberFormatException e) { + throw parseErrorHere("Expected integer but got " + e.getMessage()); + } + } - public JsonValueReader setParsers(IFunction parsers) { - if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { - // ignore the default parser - parsers = null; + private IValue inferNullValue(Map nulls, Type expected) { + return nulls.keySet().stream() + .sorted((x,y) -> x.compareTo(y)) // smaller types are matched first + .filter(superType -> expected.isSubtypeOf(superType)) // remove any type that does not fit + .findFirst() // give the most specific match + .map(t -> nulls.get(t)) // lookup the corresponding null value + .filter(r -> r.getType().isSubtypeOf(expected)) // the value in the table still has to fit the currently expected type + .orElse(null); // or muddle on and throw NPE elsewhere + + // The NPE triggering "elsewhere" should help with fault localization. } - this.parsers = parsers; - return this; - } + @Override + public IValue visitExternal(Type type) throws IOException { + throw parseErrorHere("External type " + type + "is not implemented yet by the json reader:" + in.getPath()); + } - /** - * Read and validate a Json stream as an IValue - * @param in json stream - * @param expected type to validate against (recursively) - * @return an IValue of the expected type - * @throws IOException when either a parse error or a validation error occurs - */ - public IValue read(JsonReader in, Type expected) throws IOException { - IValue res = expected.accept(new ITypeVisitor() { - @Override - public IValue visitInteger(Type type) throws IOException { - try { - switch (in.peek()) { - case NUMBER: - return vf.integer(in.nextLong()); - case STRING: - return vf.integer(in.nextString()); - case NULL: - in.nextNull(); - return inferNullValue(nulls, type); - default: - throw new IOException("Expected integer but got " + in.peek()); - } - } - catch (NumberFormatException e) { - throw new IOException("Expected integer but got " + e.getMessage()); - } + @Override + public IValue visitString(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); } - public IValue visitReal(Type type) throws IOException { - try { - switch (in.peek()) { - case NUMBER: - return vf.real(in.nextDouble()); - case STRING: - return vf.real(in.nextString()); - case NULL: - in.nextNull(); - return inferNullValue(nulls, type); - default: - throw new IOException("Expected integer but got " + in.peek()); - } + return vf.string(in.nextString()); + } + + @Override + public IValue visitTuple(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); + } + + List l = new ArrayList<>(); + in.beginArray(); + + if (type.hasFieldNames()) { + for (int i = 0; i < type.getArity(); i++) { + l.add(read(in, type.getFieldType(i))); } - catch (NumberFormatException e) { - throw new IOException("Expected integer but got " + e.getMessage()); + } + else { + for (int i = 0; i < type.getArity(); i++) { + l.add(read(in, type.getFieldType(i))); } } - - private IValue inferNullValue(Map nulls, Type expected) { - return nulls.keySet().stream() - .sorted((x,y) -> x.compareTo(y)) // smaller types are matched first - .filter(superType -> expected.isSubtypeOf(superType)) // remove any type that does not fit - .findFirst() // give the most specific match - .map(t -> nulls.get(t)) // lookup the corresponding null value - .filter(r -> r.getType().isSubtypeOf(expected)) // the value in the table still has to fit the currently expected type - .orElse(null); // or muddle on and throw NPE elsewhere - // The NPE triggering "elsewhere" should help with fault localization. - } + in.endArray(); + return vf.tuple(l.toArray(new IValue[l.size()])); + } + + @Override + public IValue visitVoid(Type type) throws IOException { + throw parseErrorHere("Can not read json values of type void: " + in.getPath()); + } - @Override - public IValue visitExternal(Type type) throws IOException { - throw new IOException("External type " + type + "is not implemented yet by the json reader:" + in.getPath()); + @Override + public IValue visitFunction(Type type) throws IOException { + throw parseErrorHere("Can not read json values of function types: " + in.getPath()); + } + + @Override + public IValue visitSourceLocation(Type type) throws IOException { + switch (in.peek()) { + case STRING: + return sourceLocationString(); + case BEGIN_OBJECT: + return sourceLocationObject(); + default: + throw parseErrorHere("Could not find string or source location object here: " + in.getPath()); } + } + + private IValue sourceLocationObject() throws IOException { + String scheme = null; + String authority = null; + String path = null; + String fragment = ""; + String query = ""; + int offset = -1; + int length = -1; + int beginLine = -1; + int endLine = -1; + int beginColumn = -1; + int endColumn = -1; - @Override - public IValue visitString(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - - return vf.string(in.nextString()); + in.beginObject(); + + while (in.hasNext()) { + String name = in.nextName(); + switch (name) { + case "scheme": + scheme = in.nextString(); + break; + case "authority": + authority = in.nextString(); + break; + case "path": + path = in.nextString(); + break; + case "fragment": + fragment = in.nextString(); + break; + case "query": + query = in.nextString(); + break; + case "offset": + offset = in.nextInt(); + break; + case "length": + length = in.nextInt(); + break; + case "start": + case "begin": + in.beginArray(); + beginLine = in.nextInt(); + beginColumn = in.nextInt(); + in.endArray(); + break; + case "end": + in.beginArray(); + endLine = in.nextInt(); + endColumn = in.nextInt(); + in.endArray(); + break; + default: + throw parseErrorHere("unexpected property name " + name + " :" + in.getPath()); + } } - @Override - public IValue visitTuple(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); + in.endObject(); + try { + ISourceLocation root; + if (scheme != null && authority != null && query != null && fragment != null) { + root = vf.sourceLocation(scheme, authority, path, query, fragment); } - - List l = new ArrayList<>(); - in.beginArray(); - - if (type.hasFieldNames()) { - for (int i = 0; i < type.getArity(); i++) { - l.add(read(in, type.getFieldType(i))); - } + else if (scheme != null) { + root = vf.sourceLocation(scheme, authority == null ? "" : authority, path); + } + else if (path != null) { + root = URIUtil.createFileLocation(path); } else { - for (int i = 0; i < type.getArity(); i++) { - l.add(read(in, type.getFieldType(i))); - } + throw parseErrorHere("Could not parse complete source location: " + in.getPath()); } - - in.endArray(); - return vf.tuple(l.toArray(new IValue[l.size()])); - } - - @Override - public IValue visitVoid(Type type) throws IOException { - throw new IOException("Can not read json values of type void: " + in.getPath()); + if (offset != -1 && length != -1 && beginLine != -1 && endLine != -1 && beginColumn != -1 && endColumn != -1) { + return vf.sourceLocation(root, offset, length, beginLine, endLine, beginColumn, endColumn); + } + if (offset != -1 && length != -1) { + return vf.sourceLocation(root, offset, length); + } + return root; + } catch (URISyntaxException e) { + throw parseErrorHere(e.getMessage()); } + } - @Override - public IValue visitFunction(Type type) throws IOException { - throw new IOException("Can not read json values of function types: " + in.getPath()); + @Override + public IValue visitValue(Type type) throws IOException { + switch (in.peek()) { + case NUMBER: + try { + return vf.integer(in.nextLong()); + } catch (NumberFormatException e) { + return vf.real(in.nextDouble()); + } + case STRING: + return visitString(TF.stringType()); + case BEGIN_ARRAY: + return visitList(TF.listType(TF.valueType())); + case BEGIN_OBJECT: + return visitNode(TF.nodeType()); + case BOOLEAN: + return visitBool(TF.nodeType()); + case NAME: + // this would be weird though + return vf.string(in.nextName()); + case NULL: + in.nextNull(); + return inferNullValue(nulls, type); + default: + throw parseErrorHere("Did not expect end of Json value here, while looking for " + type + " + at " + in.getPath()); } - - @Override - public IValue visitSourceLocation(Type type) throws IOException { - switch (in.peek()) { - case STRING: - return sourceLocationString(); - case BEGIN_OBJECT: - return sourceLocationObject(); - default: - throw new IOException("Could not find string or source location object here: " + in.getPath()); - } - } - - private IValue sourceLocationObject() throws IOException { - String scheme = null; - String authority = null; - String path = null; - String fragment = ""; - String query = ""; - int offset = -1; - int length = -1; - int beginLine = -1; - int endLine = -1; - int beginColumn = -1; - int endColumn = -1; - - in.beginObject(); - - while (in.hasNext()) { - String name = in.nextName(); - switch (name) { - case "scheme": - scheme = in.nextString(); - break; - case "authority": - authority = in.nextString(); - break; - case "path": - path = in.nextString(); - break; - case "fragment": - fragment = in.nextString(); - break; - case "query": - query = in.nextString(); - break; - case "offset": - offset = in.nextInt(); - break; - case "length": - length = in.nextInt(); - break; - case "start": - case "begin": - in.beginArray(); - beginLine = in.nextInt(); - beginColumn = in.nextInt(); - in.endArray(); - break; - case "end": - in.beginArray(); - endLine = in.nextInt(); - endColumn = in.nextInt(); - in.endArray(); - break; - default: - throw new IOException("unexpected property name " + name + " :" + in.getPath()); - } - } - - in.endObject(); + } + + private IValue sourceLocationString() throws IOException { try { - ISourceLocation root; - if (scheme != null && authority != null && query != null && fragment != null) { - root = vf.sourceLocation(scheme, authority, path, query, fragment); - } - else if (scheme != null) { - root = vf.sourceLocation(scheme, authority == null ? "" : authority, path); + String val = in.nextString().trim(); + + if (val.startsWith("|") && (val.endsWith("|") || val.endsWith(")"))) { + return new StandardTextReader().read(vf, new StringReader(val)); } - else if (path != null) { - root = URIUtil.createFileLocation(path); + else if (val.contains("://")) { + return vf.sourceLocation(URIUtil.createFromEncoded(val)); } else { - throw new IOException("Could not parse complete source location: " + in.getPath()); - } - if (offset != -1 && length != -1 && beginLine != -1 && endLine != -1 && beginColumn != -1 && endColumn != -1) { - return vf.sourceLocation(root, offset, length, beginLine, endLine, beginColumn, endColumn); + // will be simple interpreted as an absolute file name + return URIUtil.createFileLocation(val); } - if (offset != -1 && length != -1) { - return vf.sourceLocation(root, offset, length); - } - return root; - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public IValue visitValue(Type type) throws IOException { - switch (in.peek()) { - case NUMBER: - try { - return vf.integer(in.nextLong()); - } catch (NumberFormatException e) { - return vf.real(in.nextDouble()); - } - case STRING: - return visitString(TF.stringType()); - case BEGIN_ARRAY: - return visitList(TF.listType(TF.valueType())); - case BEGIN_OBJECT: - return visitNode(TF.nodeType()); - case BOOLEAN: - return visitBool(TF.nodeType()); - case NAME: - // this would be weird though - return vf.string(in.nextName()); - case NULL: - in.nextNull(); - return inferNullValue(nulls, type); - default: - throw new IOException("Did not expect end of Json value here, while looking for " + type + " + at " + in.getPath()); + } + catch (URISyntaxException e) { + throw parseErrorHere(e.getMessage()); } + } + + public IValue visitRational(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); } - private IValue sourceLocationString() throws IOException { - try { - String val = in.nextString().trim(); - - if (val.startsWith("|") && (val.endsWith("|") || val.endsWith(")"))) { - return new StandardTextReader().read(vf, new StringReader(val)); + switch (in.peek()) { + case BEGIN_OBJECT: + in.beginObject(); + IInteger nomO = null, denomO = null; + while (in.hasNext()) { + switch (in.nextName()) { + case "nominator": + nomO = (IInteger) read(in, TF.integerType()); + case "denominator": + denomO = (IInteger) read(in, TF.integerType()); } - else if (val.contains("://")) { - return vf.sourceLocation(URIUtil.createFromEncoded(val)); - } - else { - // will be simple interpreted as an absolute file name - return URIUtil.createFileLocation(val); - } - } catch (URISyntaxException e) { - throw new IOException("could not parse URI:" + in.getPath() + " due to " + e.getMessage(), e); } - } - - public IValue visitRational(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - - switch (in.peek()) { - case BEGIN_OBJECT: - in.beginObject(); - IInteger nomO = null, denomO = null; - while (in.hasNext()) { - switch (in.nextName()) { - case "nominator": - nomO = (IInteger) read(in, TF.integerType()); - case "denominator": - denomO = (IInteger) read(in, TF.integerType()); - } - } - in.endObject(); + in.endObject(); - if (nomO == null || denomO == null) { - throw new IOException("Did not find all fields of expected rational at " + in.getPath()); - } + if (nomO == null || denomO == null) { + throw parseErrorHere("Did not find all fields of expected rational at " + in.getPath()); + } - return vf.rational(nomO, denomO); - case BEGIN_ARRAY: - in.beginArray(); - IInteger nomA = (IInteger) read(in, TF.integerType()); - IInteger denomA = (IInteger) read(in, TF.integerType()); - in.endArray(); - return vf.rational(nomA, denomA); - case STRING: - return vf.rational(in.nextString()); - default: - throw new IOException("Expected integer but got " + in.peek()); - } + return vf.rational(nomO, denomO); + case BEGIN_ARRAY: + in.beginArray(); + IInteger nomA = (IInteger) read(in, TF.integerType()); + IInteger denomA = (IInteger) read(in, TF.integerType()); + in.endArray(); + return vf.rational(nomA, denomA); + case STRING: + return vf.rational(in.nextString()); + default: + throw parseErrorHere("Expected integer but got " + in.peek()); + } + } + + @Override + public IValue visitMap(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); } + IMapWriter w = vf.mapWriter(); - @Override - public IValue visitMap(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - IMapWriter w = vf.mapWriter(); - - switch (in.peek()) { - case BEGIN_OBJECT: - in.beginObject(); - if (!type.getKeyType().isString()) { - throw new IOException("Can not read JSon object as a map if the key type of the map (" + type + ") is not a string at " + in.getPath()); - } - - while (in.hasNext()) { - w.put(vf.string(in.nextName()), read(in, type.getValueType())); - } - in.endObject(); - return w.done(); - case BEGIN_ARRAY: + switch (in.peek()) { + case BEGIN_OBJECT: + in.beginObject(); + if (!type.getKeyType().isString()) { + throw parseErrorHere("Can not read JSon object as a map if the key type of the map (" + type + ") is not a string at " + in.getPath()); + } + + while (in.hasNext()) { + w.put(vf.string(in.nextName()), read(in, type.getValueType())); + } + in.endObject(); + return w.done(); + case BEGIN_ARRAY: + in.beginArray(); + while (in.hasNext()) { in.beginArray(); - while (in.hasNext()) { - in.beginArray(); - IValue key = read(in, type.getKeyType()); - IValue value = read(in, type.getValueType()); - w.put(key,value); - in.endArray(); - } + IValue key = read(in, type.getKeyType()); + IValue value = read(in, type.getValueType()); + w.put(key,value); in.endArray(); - return w.done(); - default: - throw new IOException("Expected a map encoded as an object or an nested array to match " + type); - } - } - - @Override - public IValue visitAlias(Type type) throws IOException { - while (type.isAliased()) { - type = type.getAliased(); - } - - return type.accept(this); + } + in.endArray(); + return w.done(); + default: + throw parseErrorHere("Expected a map encoded as an object or an nested array to match " + type); } - - @Override - public IValue visitBool(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - return vf.bool(in.nextBoolean()); + } + + @Override + public IValue visitAlias(Type type) throws IOException { + while (type.isAliased()) { + type = type.getAliased(); } - private int getPos() { - if (src == null) { - return 0; - } + return type.accept(this); + } - try { - return (int) posHandler.get(in) - 1; - } - catch (IllegalArgumentException | SecurityException e) { - // stop trying to recover the positions - src = null; - return 0; - } + @Override + public IValue visitBool(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); } + return vf.bool(in.nextBoolean()); + } - private int getLine() { - if (src == null) { - return 0; - } + private int getPos() { + try { + return Math.max(0, (int) posHandler.get(in) - 1); + } + catch (IllegalArgumentException | SecurityException e) { + // stop trying to recover the positions + src = null; + return 0; + } + } - try { - return (int) lineHandler.get(in) + 1; - } - catch (IllegalArgumentException | SecurityException e) { - // stop trying to recover the positions - src = null; - return 0; - } + private int getLine() { + if (src == null) { + return 0; } - private int getCol() { - if (src == null) { - return 0; - } + try { + return (int) lineHandler.get(in) + 1; + } + catch (IllegalArgumentException | SecurityException e) { + // stop trying to recover the positions + src = null; + return 0; + } + } - try { - return getPos() - (int) lineStartHandler.get(in); - } - catch (IllegalArgumentException | SecurityException e) { - // stop trying to recover the positions - src = null; - return 0; - } + private int getCol() { + if (src == null) { + return 0; } - /** - * Expecting an ADT we found NULL on the lookahead. - * This is either a Maybe or we can use the map - * of null values. - */ - private IValue visitNullAsAbstractData(Type type) { - return inferNullValue(nulls, type); + try { + return getPos() - (int) lineStartHandler.get(in); } + catch (IllegalArgumentException | SecurityException e) { + // stop trying to recover the positions + src = null; + return 0; + } + } - /** - * Expecting an ADT we found a string value instead. - * Now we can (try to) apply the parsers that were passed in. - * If that does not fly, we can interpret strings as nullary ADT - * constructors. - */ - private IValue visitStringAsAbstractData(Type type) throws IOException { - var stringInput = in.nextString(); + protected Throw parseErrorHere(String cause) { + var location = src == null ? URIUtil.rootLocation("unknown") : src; + int offset = getPos(); + int line = getLine(); + int col = getCol(); + + return RuntimeExceptionFactory.jsonParseError( + vf.sourceLocation(location, offset, 1, line, line, col, col + 1), + cause, + in.getPath()); + } - // might be a parsable string. let's see. - if (parsers != null) { - var reified = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); - - try { - return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); - } - catch (Throw t) { - Type excType = t.getException().getType(); + /** + * Expecting an ADT we found NULL on the lookahead. + * This is either a Maybe or we can use the map + * of null values. + */ + private IValue visitNullAsAbstractData(Type type) { + return inferNullValue(nulls, type); + } - if (excType.isAbstractData() && ((IConstructor) t.getException()).getConstructorType().getName().equals("ParseError")) { - throw new IOException(t); // an actual parse error is meaningful to report - } - // otherwise we fall through to enum recognition - } + /** + * Expecting an ADT we found a string value instead. + * Now we can (try to) apply the parsers that were passed in. + * If that does not fly, we can interpret strings as nullary ADT + * constructors. + */ + private IValue visitStringAsAbstractData(Type type) throws IOException { + var stringInput = in.nextString(); + + // might be a parsable string. let's see. + if (parsers != null) { + var reified = new org.rascalmpl.types.TypeReifier(vf).typeToValue(type, new TypeStore(), vf.map()); + + try { + return parsers.call(Collections.emptyMap(), reified, vf.string(stringInput)); } + catch (Throw t) { + Type excType = t.getException().getType(); - // enum! - Set enumCons = store.lookupConstructor(type, stringInput); - - for (Type candidate : enumCons) { - if (candidate.getArity() == 0) { - return vf.constructor(candidate); + if (excType.isAbstractData() && ((IConstructor) t.getException()).getConstructorType().getName().equals("ParseError")) { + throw t; // that's a real parse error to report } - } - - if (parsers != null) { - throw new IOException("parser failed to recognize \"" + stringInput + "\" and no nullary constructor found for " + type + "either"); - } - else { - throw new IOException("no nullary constructor found for " + type + ", that matches " + stringInput); + // otherwise we fall through to enum recognition } } - /** - * This is the main workhorse. Every object is mapped one-to-one to an ADT constructor - * instance. The field names (keyword parameters and positional) are mapped to field - * names of the object. The name of the constructor is _not_ consequential. - * @param type - * @return - * @throws IOException - */ - private IValue visitObjectAsAbstractData(Type type) throws IOException { - Set alternatives = store.lookupAlternatives(type); - if (alternatives.size() > 1) { - monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); + // enum! + Set enumCons = store.lookupConstructor(type, stringInput); + + for (Type candidate : enumCons) { + if (candidate.getArity() == 0) { + return vf.constructor(candidate); } - Type cons = alternatives.iterator().next(); - - in.beginObject(); - int startPos = getPos(); - int startLine = getLine(); - int startCol = getCol(); - - IValue[] args = new IValue[cons.getArity()]; - Map kwParams = new HashMap<>(); - - if (!cons.hasFieldNames() && cons.getArity() != 0) { - throw new IOException("For the object encoding constructors must have field names " + in.getPath()); + } + + if (parsers != null) { + throw parseErrorHere("parser failed to recognize \"" + stringInput + "\" and no nullary constructor found for " + type + "either"); + } + else { + throw parseErrorHere("no nullary constructor found for " + type + ", that matches " + stringInput); + } + } + + /** + * This is the main workhorse. Every object is mapped one-to-one to an ADT constructor + * instance. The field names (keyword parameters and positional) are mapped to field + * names of the object. The name of the constructor is _not_ consequential. + * @param type + * @return + * @throws IOException + */ + private IValue visitObjectAsAbstractData(Type type) throws IOException { + Set alternatives = store.lookupAlternatives(type); + if (alternatives.size() > 1) { + monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); + } + Type cons = alternatives.iterator().next(); + + in.beginObject(); + int startPos = getPos(); + int startLine = getLine(); + int startCol = getCol(); + + IValue[] args = new IValue[cons.getArity()]; + Map kwParams = new HashMap<>(); + + if (!cons.hasFieldNames() && cons.getArity() != 0) { + throw parseErrorHere("For the object encoding constructors must have field names " + in.getPath()); + } + + while (in.hasNext()) { + String label = in.nextName(); + if (cons.hasField(label)) { + IValue val = read(in, cons.getFieldType(label)); + if (val != null) { + args[cons.getFieldIndex(label)] = val; + } + else { + throw parseErrorHere("Could not parse argument " + label + ":" + in.getPath()); + } } - - while (in.hasNext()) { - String label = in.nextName(); - if (cons.hasField(label)) { - IValue val = read(in, cons.getFieldType(label)); + else if (cons.hasKeywordField(label, store)) { + if (!isNull()) { // lookahead for null to give default parameters the preference. + IValue val = read(in, store.getKeywordParameterType(cons, label)); + // null can still happen if the nulls map doesn't have a default if (val != null) { - args[cons.getFieldIndex(label)] = val; - } - else { - throw new IOException("Could not parse argument " + label + ":" + in.getPath()); + // if the value is null we'd use the default value of the defined field in the constructor + kwParams.put(label, val); } } - else if (cons.hasKeywordField(label, store)) { - if (!isNull()) { // lookahead for null to give default parameters the preference. - IValue val = read(in, store.getKeywordParameterType(cons, label)); - // null can still happen if the nulls map doesn't have a default - if (val != null) { - // if the value is null we'd use the default value of the defined field in the constructor - kwParams.put(label, val); - } - } - } - else { // its a normal arg, pass its label to the child - throw new IOException("Unknown field " + label + ":" + in.getPath()); - } } - - in.endObject(); - int endPos = getPos(); - int endLine = getLine(); - int endCol = getCol(); - - for (int i = 0; i < args.length; i++) { - if (args[i] == null) { - throw new IOException("Missing argument " + cons.getFieldName(i) + " to " + cons + ":" + in.getPath()); - } + else { // its a normal arg, pass its label to the child + throw parseErrorHere("Unknown field " + label + ":" + in.getPath()); } - - if (src != null) { - kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + } + + in.endObject(); + int endPos = getPos(); + int endLine = getLine(); + int endCol = getCol(); + + for (int i = 0; i < args.length; i++) { + if (args[i] == null) { + throw parseErrorHere("Missing argument " + cons.getFieldName(i) + " to " + cons + ":" + in.getPath()); } + } + + if (src != null) { + kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + } - return vf.constructor(cons, args, kwParams); - } + return vf.constructor(cons, args, kwParams); + } - @Override - public IValue visitAbstractData(Type type) throws IOException { - if (UtilMaybe.isMaybe(type)) { - if (in.peek() == JsonToken.NULL) { - in.nextNull(); - return UtilMaybe.nothing(); - } - else { - // dive into the wrapped type, and wrap the result. Could be a str, int, or anything. - return UtilMaybe.just(type.getTypeParameters().getFieldType(0).accept(this)); - } + @Override + public IValue visitAbstractData(Type type) throws IOException { + if (UtilMaybe.isMaybe(type)) { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return UtilMaybe.nothing(); } - - switch (in.peek()) { - case NULL: - return visitNullAsAbstractData(type); - case STRING: - return visitStringAsAbstractData(type); - case BEGIN_OBJECT: - return visitObjectAsAbstractData(type); - default: - throw new IOException("Expected ADT:" + type + ", but found " + in.peek().toString()); + else { + // dive into the wrapped type, and wrap the result. Could be a str, int, or anything. + return UtilMaybe.just(type.getTypeParameters().getFieldType(0).accept(this)); } } - - @Override - public IValue visitConstructor(Type type) throws IOException { - return read(in, type.getAbstractDataType()); + + switch (in.peek()) { + case NULL: + return visitNullAsAbstractData(type); + case STRING: + return visitStringAsAbstractData(type); + case BEGIN_OBJECT: + return visitObjectAsAbstractData(type); + default: + throw parseErrorHere("Expected ADT:" + type + ", but found " + in.peek().toString()); } - - @Override - public IValue visitNode(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } + } - in.beginObject(); - int startPos = getPos(); - int startLine = getLine(); - int startCol = getCol(); - - Map kws = new HashMap<>(); - - while (in.hasNext()) { - String kwName = in.nextName(); - IValue value = read(in, TF.valueType()); - - if (value != null) { - kws.put(kwName, value); - } - } - - in.endObject(); - int endPos = getPos(); - int endLine = getLine(); - int endCol = getCol(); + @Override + public IValue visitConstructor(Type type) throws IOException { + return read(in, type.getAbstractDataType()); + } - if (src != null) { - kws.put(kws.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); - } - - return vf.node("object", new IValue[] { }, kws); + @Override + public IValue visitNode(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); } + + in.beginObject(); + int startPos = getPos(); + int startLine = getLine(); + int startCol = getCol(); + + Map kws = new HashMap<>(); - @Override - public IValue visitNumber(Type type) throws IOException { - return visitInteger(type); + while (in.hasNext()) { + String kwName = in.nextName(); + IValue value = read(in, TF.valueType()); + + if (value != null) { + kws.put(kwName, value); + } } - @Override - public IValue visitParameter(Type type) throws IOException { - return type.getBound().accept(this); + in.endObject(); + int endPos = getPos(); + int endLine = getLine(); + int endCol = getCol(); + + if (src != null) { + kws.put(kws.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } - @Override - public IValue visitDateTime(Type type) throws IOException { - try { - switch (in.peek()) { - case STRING: - Date parsedDate = format.get().parse(in.nextString()); - return vf.datetime(parsedDate.toInstant().toEpochMilli()); - case NUMBER: - return vf.datetime(in.nextLong()); - default: - throw new IOException("Expected a datetime instant " + in.getPath()); - } - } catch (ParseException e) { - throw new IOException("Could not parse date: " + in.getPath()); + return vf.node("object", new IValue[] { }, kws); + } + + @Override + public IValue visitNumber(Type type) throws IOException { + return visitInteger(type); + } + + @Override + public IValue visitParameter(Type type) throws IOException { + return type.getBound().accept(this); + } + + @Override + public IValue visitDateTime(Type type) throws IOException { + try { + switch (in.peek()) { + case STRING: + Date parsedDate = format.get().parse(in.nextString()); + return vf.datetime(parsedDate.toInstant().toEpochMilli()); + case NUMBER: + return vf.datetime(in.nextLong()); + default: + throw parseErrorHere("Expected a datetime instant " + in.getPath()); } + } catch (ParseException e) { + throw parseErrorHere("Could not parse date: " + in.getPath()); } - - @Override - public IValue visitList(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } + } - IListWriter w = vf.listWriter(); - in.beginArray(); - while (in.hasNext()) { - // here we pass label from the higher context - w.append(read(in, type.getElementType())); - } + @Override + public IValue visitList(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); + } - in.endArray(); - return w.done(); + IListWriter w = vf.listWriter(); + in.beginArray(); + while (in.hasNext()) { + // here we pass label from the higher context + w.append(read(in, type.getElementType())); } - - public IValue visitSet(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - ISetWriter w = vf.setWriter(); - in.beginArray(); - while (in.hasNext()) { - // here we pass label from the higher context - w.insert(read(in, type.getElementType())); - } + in.endArray(); + return w.done(); + } + + public IValue visitSet(Type type) throws IOException { + if (isNull()) { + return inferNullValue(nulls, type); + } - in.endArray(); - return w.done(); + ISetWriter w = vf.setWriter(); + in.beginArray(); + while (in.hasNext()) { + // here we pass label from the higher context + w.insert(read(in, type.getElementType())); } - private boolean isNull() throws IOException { - // we use null in JSon to encode optional values. - // this will be mapped to keyword parameters in Rascal, - // or an exception if we really need a value - if (in.peek() == JsonToken.NULL) { - in.nextNull(); - return true; - } - return false; + in.endArray(); + return w.done(); + } + + private boolean isNull() throws IOException { + // we use null in JSon to encode optional values. + // this will be mapped to keyword parameters in Rascal, + // or an exception if we really need a value + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return true; + } + return false; + } + } + + private static final TypeFactory TF = TypeFactory.getInstance(); + private final TypeStore store; + private final IRascalValueFactory vf; + private ThreadLocal format; + private final IRascalMonitor monitor; + private ISourceLocation src; + private VarHandle posHandler; + private VarHandle lineHandler; + private VarHandle lineStartHandler; + private IFunction parsers; + private Map nulls = Collections.emptyMap(); + + + /** + * @param vf factory which will be used to construct values + * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields + */ + public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { + this.vf = vf; + this.store = store; + this.monitor = monitor; + this.src = (src == null) ? URIUtil.rootLocation("unknown") : src; + + setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + + // this is for origin tracking as well as accurate parse errors + try { + var lookup = MethodHandles.lookup(); + var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); + this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); + this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); + this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + } + catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + // we disable the origin tracking if we can not get to the fields + src = null; + monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + } + } + + public JsonValueReader(IRascalValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { + this(vf, new TypeStore(), monitor, src); + } + + public JsonValueReader setNulls(Map nulls) { + this.nulls = nulls; + return this; + } + /** + * Builder method to set the format to use for all date-time values encoded as strings + */ + public JsonValueReader setCalendarFormat(String format) { + // SimpleDateFormat is not thread safe, so here we make sure + // we can use objects of this reader in different threads at the same time + this.format = new ThreadLocal() { + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat(format); } + }; + return this; + } + + public JsonValueReader setParsers(IFunction parsers) { + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { + // ignore the default parser + parsers = null; + } - }); - - return res; + this.parsers = parsers; + return this; } + /** + * Read and validate a Json stream as an IValue + * @param in json stream + * @param expected type to validate against (recursively) + * @return an IValue of the expected type + * @throws IOException when either a parse error or a validation error occurs + */ + public IValue read(JsonReader in, Type expected) throws IOException { + var dispatch = new ExpectedTypeDispatcher(in); + try { + return expected.accept(dispatch); + } + catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException | NullPointerException e) { + throw dispatch.parseErrorHere(e.getMessage()); + } + } } \ No newline at end of file From 50b7fe6beec10254153ccdb4108ec8b49ffb8692 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 12:03:41 +0200 Subject: [PATCH 30/45] can still turn off origin tracking --- .../lang/json/internal/JsonValueReader.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 65e6c0e3c85..bde3e95a654 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.lang.ObjectUtils.Null; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; @@ -52,7 +51,6 @@ import io.usethesource.vallang.type.TypeStore; import com.google.gson.JsonParseException; -import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.MalformedJsonException; @@ -404,6 +402,10 @@ public IValue visitBool(Type type) throws IOException { } private int getPos() { + if (src == null) { + return 0; + } + try { return Math.max(0, (int) posHandler.get(in) - 1); } @@ -572,7 +574,7 @@ else if (cons.hasKeywordField(label, store)) { } } - if (src != null) { + if (originTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } @@ -637,7 +639,7 @@ public IValue visitNode(Type type) throws IOException { int endLine = getLine(); int endCol = getCol(); - if (src != null) { + if (originTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } @@ -722,6 +724,7 @@ private boolean isNull() throws IOException { private ThreadLocal format; private final IRascalMonitor monitor; private ISourceLocation src; + private boolean originTracking; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; @@ -738,6 +741,7 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m this.store = store; this.monitor = monitor; this.src = (src == null) ? URIUtil.rootLocation("unknown") : src; + setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); @@ -748,9 +752,11 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + this.originTracking = (src != null); } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { // we disable the origin tracking if we can not get to the fields + originTracking = false; src = null; monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); } From 149e5462a7f6d8ef059144eaf0f409f0a772a1b5 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 12:04:46 +0200 Subject: [PATCH 31/45] better error message --- src/org/rascalmpl/library/lang/json/IO.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index cc8c4929888..6c216dc218a 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -126,8 +126,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } catch (NullPointerException e) { - e.printStackTrace(); - throw RuntimeExceptionFactory.io(values.string("NPE"), null, null); + throw RuntimeExceptionFactory.io(values.string("NPE in error handling code"), null, null); } } From f32eb53af6a3314bc655960cf663469bb88d8936 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 27 Jun 2024 15:23:35 +0200 Subject: [PATCH 32/45] improved error reporting around quotes names with unbalanced quotes, and quoted strings with unbalanced quotes --- src/org/rascalmpl/library/lang/json/IO.rsc | 2 +- .../lang/json/internal/JsonValueReader.java | 67 +++++++++++++------ 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index ac97e2762e6..c49a54b46ff 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -72,7 +72,7 @@ public map[type[value] forType, value nullValue] defaultJSONNULLValues = ( #node : "null"(), #int : -1, #real : -1.0, - #rat :-1r1, + #rat : -1r1, #value : "null"(), #str : "", #list[value] : [], diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index bde3e95a654..d9b603bbadb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -63,11 +63,34 @@ public class JsonValueReader { private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; + private int lastPos; private ExpectedTypeDispatcher(JsonReader in) { this.in = in; } + /** + * Wrapping in.nextString for better error reporting + */ + private String nextString() throws IOException { + // need to cache the last position before parsing the string, because + // when the string does not have a balancing quote the read accidentally + // rewinds the position to 0. + lastPos=getPos(); + return in.nextString(); + } + + /** + * Wrapping in.nextName for better error reporting + */ + private String nextName() throws IOException { + // need to cache the last position before parsing the string, because + // when the string does not have a balancing quote the read accidentally + // rewinds the position to 0. + lastPos=getPos(); + return in.nextName(); + } + @Override public IValue visitInteger(Type type) throws IOException { try { @@ -75,7 +98,7 @@ public IValue visitInteger(Type type) throws IOException { case NUMBER: return vf.integer(in.nextLong()); case STRING: - return vf.integer(in.nextString()); + return vf.integer(nextString()); case NULL: in.nextNull(); return inferNullValue(nulls, type); @@ -94,7 +117,7 @@ public IValue visitReal(Type type) throws IOException { case NUMBER: return vf.real(in.nextDouble()); case STRING: - return vf.real(in.nextString()); + return vf.real(nextString()); case NULL: in.nextNull(); return inferNullValue(nulls, type); @@ -129,8 +152,8 @@ public IValue visitString(Type type) throws IOException { if (isNull()) { return inferNullValue(nulls, type); } - - return vf.string(in.nextString()); + + return vf.string(nextString()); } @Override @@ -195,22 +218,23 @@ private IValue sourceLocationObject() throws IOException { in.beginObject(); while (in.hasNext()) { - String name = in.nextName(); + String name = nextName(); + switch (name) { case "scheme": - scheme = in.nextString(); + scheme = nextString(); break; case "authority": - authority = in.nextString(); + authority = nextString(); break; case "path": - path = in.nextString(); + path = nextString(); break; case "fragment": - fragment = in.nextString(); + fragment = nextString(); break; case "query": - query = in.nextString(); + query = nextString(); break; case "offset": offset = in.nextInt(); @@ -282,7 +306,7 @@ public IValue visitValue(Type type) throws IOException { return visitBool(TF.nodeType()); case NAME: // this would be weird though - return vf.string(in.nextName()); + return vf.string(nextName()); case NULL: in.nextNull(); return inferNullValue(nulls, type); @@ -293,7 +317,7 @@ public IValue visitValue(Type type) throws IOException { private IValue sourceLocationString() throws IOException { try { - String val = in.nextString().trim(); + String val = nextString().trim(); if (val.startsWith("|") && (val.endsWith("|") || val.endsWith(")"))) { return new StandardTextReader().read(vf, new StringReader(val)); @@ -321,7 +345,7 @@ public IValue visitRational(Type type) throws IOException { in.beginObject(); IInteger nomO = null, denomO = null; while (in.hasNext()) { - switch (in.nextName()) { + switch (nextName()) { case "nominator": nomO = (IInteger) read(in, TF.integerType()); case "denominator": @@ -343,7 +367,7 @@ public IValue visitRational(Type type) throws IOException { in.endArray(); return vf.rational(nomA, denomA); case STRING: - return vf.rational(in.nextString()); + return vf.rational(nextString()); default: throw parseErrorHere("Expected integer but got " + in.peek()); } @@ -364,7 +388,7 @@ public IValue visitMap(Type type) throws IOException { } while (in.hasNext()) { - w.put(vf.string(in.nextName()), read(in, type.getValueType())); + w.put(vf.string(nextName()), read(in, type.getValueType())); } in.endObject(); return w.done(); @@ -448,12 +472,12 @@ private int getCol() { protected Throw parseErrorHere(String cause) { var location = src == null ? URIUtil.rootLocation("unknown") : src; - int offset = getPos(); + int offset = Math.max(getPos(), lastPos); int line = getLine(); int col = getCol(); return RuntimeExceptionFactory.jsonParseError( - vf.sourceLocation(location, offset, 1, line, line, col, col + 1), + vf.sourceLocation(location,offset, 1, line, line, col, col + 1), cause, in.getPath()); } @@ -474,7 +498,7 @@ private IValue visitNullAsAbstractData(Type type) { * constructors. */ private IValue visitStringAsAbstractData(Type type) throws IOException { - var stringInput = in.nextString(); + var stringInput = nextString(); // might be a parsable string. let's see. if (parsers != null) { @@ -538,7 +562,7 @@ private IValue visitObjectAsAbstractData(Type type) throws IOException { } while (in.hasNext()) { - String label = in.nextName(); + String label = nextName(); if (cons.hasField(label)) { IValue val = read(in, cons.getFieldType(label)); if (val != null) { @@ -626,7 +650,7 @@ public IValue visitNode(Type type) throws IOException { Map kws = new HashMap<>(); while (in.hasNext()) { - String kwName = in.nextName(); + String kwName = nextName(); IValue value = read(in, TF.valueType()); if (value != null) { @@ -661,7 +685,8 @@ public IValue visitDateTime(Type type) throws IOException { try { switch (in.peek()) { case STRING: - Date parsedDate = format.get().parse(in.nextString()); + lastPos = getPos(); + Date parsedDate = format.get().parse(nextString()); return vf.datetime(parsedDate.toInstant().toEpochMilli()); case NUMBER: return vf.datetime(in.nextLong()); From b6782ee53d415889120116982f9687441b65601c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 9 Jul 2024 20:12:42 +0200 Subject: [PATCH 33/45] added tests for parsing/formatting while reading/writing JSON --- src/org/rascalmpl/library/lang/json/IO.rsc | 3 +- .../tests/library/lang/json/JSONIOTests.rsc | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index c49a54b46ff..7199346b781 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -65,7 +65,8 @@ java &T readJSON( bool lenient=false, bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }, - map [type[value] forType, value nullValue] nulls = defaultJSONNULLValues); + map [type[value] forType, value nullValue] nulls = defaultJSONNULLValues +); public map[type[value] forType, value nullValue] defaultJSONNULLValues = ( #Maybe[value] : nothing(), diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 21e5c9acc61..f1785ffba12 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -58,10 +58,10 @@ test bool json3() = writeRead(#DATA3, data3(123,kw="123")); test bool json4(Enum e) = writeRead(#DATA4, data4(e=e)); test bool originTracking() { - example = readJSON(#node, |std:///lang/rascal/tests/library/lang/json/glossary.json|, trackOrigins=true); + ex2 = readJSON(#node, |std:///lang/rascal/tests/library/lang/json/glossary.json|, trackOrigins=true); content = readFile(|std:///lang/rascal/tests/library/lang/json/glossary.json|); - poss = [ | /node x := example, x.line?]; // every node has a .src field, otherwise this fails with an exception + poss = [ | /node x := ex2, x.line?]; // every node has a .src field, otherwise this fails with an exception for ( <- poss) { assert content[p.offset] == "{"; // all nodes start with a { @@ -94,5 +94,34 @@ test bool dealWithNull() { assert parseJSON(#Cons, "{\"bla\": \"foo\"}") == cons(bla="foo"); assert parseJSON(#Cons, "{\"bla\": null}") == cons(); + return true; +} + +data Example = example(Prop ex = F()); + +data Prop = T() | F() | and(Prop lhs, Prop rhs) | or(Prop lhs, Prop rhs); + +str format(T()) = "true"; +str format(F()) = "false"; +str format(and(Prop p1, Prop p2)) = " && "; +str format(or(Prop p1, Prop p2)) = " || "; + +Prop parse(type[Prop] _, "true") = T(); +Prop parse(type[Prop] _, "false") = F(); + +test bool formattingToStringsTest() { + ex1 = and(and(\T(), \F()),or(\T(), \F())); + + writeJSON(|memory://test-json/formatted.json|, example(ex=ex1), formatter=format); + source = readFile(|memory://test-json/formatted.json|); + + assert source == "{\"ex\":\"true && false && true || false\"}"; + + writeFile(|memory://test-json/printed.json|, "{\"ex\":\"true\"}"); + + Example result = readJSON(#Example, |memory://test-json/printed.json|, parser=parse); + + assert result.ex == T(); + return true; } \ No newline at end of file From 8e5e79c7d543cf9631e109bfe40ff5c8777edab8 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 9 Dec 2024 20:30:19 +0100 Subject: [PATCH 34/45] a difficult merge fix --- src/org/rascalmpl/library/lang/json/IO.java | 1 - src/org/rascalmpl/library/lang/json/IO.rsc | 12 +- .../lang/json/internal/JsonValueReader.java | 199 +++++------------- 3 files changed, 64 insertions(+), 148 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 616b2c24706..394db4623cc 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -104,7 +104,6 @@ public IValue fromJSON(IValue type, IString src) { } } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index f1302d635fe..f0b679a8bc4 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -61,9 +61,11 @@ java &T readJSON( loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, - bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false, + bool trackOrigins=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }, - map [type[value] forType, value nullValue] nulls = defaultJSONNULLValues + map [type[value] forType, value nullValue] nulls = defaultJSONNULLValues, + bool explicitConstructorNames = false, + bool explicitDataTypes = false ); public map[type[value] forType, value nullValue] defaultJSONNULLValues = ( @@ -89,10 +91,10 @@ java &T parseJSON( str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, - bool explicitConstructorNames=false, - bool explicitDataTypes=false, JSONParser[value] parser = (type[value] _, str _) { throw ""; }, - map[type[value] forType, value nullValue] nulls = defaultJSONNULLValues + map[type[value] forType, value nullValue] nulls = defaultJSONNULLValues, + bool explicitConstructorNames = false, + bool explicitDataTypes = false ); @javaClass{org.rascalmpl.library.lang.json.IO} diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index c43b60edd2e..fb7c7b5e436 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -43,6 +43,7 @@ import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; +import io.usethesource.vallang.IValueFactory; import io.usethesource.vallang.io.StandardTextReader; import io.usethesource.vallang.type.ITypeVisitor; import io.usethesource.vallang.type.Type; @@ -57,7 +58,6 @@ /** * This class streams a JSON stream directly to an IValue representation and validates the content * to a given type as declared in a given type store. See the Rascal file lang::json::IO::readJson for documentation. - */ public class JsonValueReader { private final class ExpectedTypeDispatcher implements ITypeVisitor { @@ -110,6 +110,7 @@ public IValue visitInteger(Type type) throws IOException { } } + @Override public IValue visitReal(Type type) throws IOException { try { switch (in.peek()) { @@ -128,7 +129,7 @@ public IValue visitReal(Type type) throws IOException { throw parseErrorHere("Expected integer but got " + e.getMessage()); } } - + private IValue inferNullValue(Map nulls, Type expected) { return nulls.keySet().stream() .sorted((x,y) -> x.compareTo(y)) // smaller types are matched first @@ -136,7 +137,7 @@ private IValue inferNullValue(Map nulls, Type expected) { .findFirst() // give the most specific match .map(t -> nulls.get(t)) // lookup the corresponding null value .filter(r -> r.getType().isSubtypeOf(expected)) // the value in the table still has to fit the currently expected type - .orElse(null); // or muddle on and throw NPE elsewhere + .orElse(null); // or muddle on and throw NPE elsewhere // The NPE triggering "elsewhere" should help with fault localization. } @@ -155,142 +156,6 @@ public IValue visitString(Type type) throws IOException { return vf.string(nextString()); } - @Override - public IValue visitTuple(Type type) throws IOException { - if (isNull()) { - return inferNullValue(nulls, type); - } - private static final TypeFactory TF = TypeFactory.getInstance(); - private final TypeStore store; - private final IValueFactory vf; - private ThreadLocal format; - private final IRascalMonitor monitor; - private ISourceLocation src; - private VarHandle posHandler; - private VarHandle lineHandler; - private VarHandle lineStartHandler; - private boolean explicitConstructorNames; - private boolean explicitDataTypes; - - /** - * @param vf factory which will be used to construct values - * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields - */ - public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { - this.vf = vf; - this.store = store; - this.monitor = monitor; - this.src = src; - setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - - if (src != null) { - try { - var lookup = MethodHandles.lookup(); - var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); - this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); - this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); - this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); - } - catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { - // we disable the origin tracking if we can not get to the fields - src = null; - monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); - } - } - } - - public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { - this(vf, new TypeStore(), monitor, src); - } - - /** - * Builder method to set the format to use for all date-time values encoded as strings - */ - public JsonValueReader setCalendarFormat(String format) { - // SimpleDateFormat is not thread safe, so here we make sure - // we can use objects of this reader in different threads at the same time - this.format = new ThreadLocal() { - protected SimpleDateFormat initialValue() { - return new SimpleDateFormat(format); - } - }; - return this; - } - - public JsonValueReader setExplicitConstructorNames(boolean value) { - this.explicitConstructorNames = value; - return this; - } - - public JsonValueReader setExplicitDataTypes(boolean value) { - this.explicitDataTypes = value; - if (value) { - this.explicitConstructorNames = true; - } - return this; - } - - - /** - * Read and validate a Json stream as an IValue - * @param in json stream - * @param expected type to validate against (recursively) - * @return an IValue of the expected type - * @throws IOException when either a parse error or a validation error occurs - */ - public IValue read(JsonReader in, Type expected) throws IOException { - IValue res = expected.accept(new ITypeVisitor() { - @Override - public IValue visitInteger(Type type) throws IOException { - try { - switch (in.peek()) { - case NUMBER: - return vf.integer(in.nextLong()); - case STRING: - return vf.integer(in.nextString()); - case NULL: - return null; - default: - throw new IOException("Expected integer but got " + in.peek()); - } - } - catch (NumberFormatException e) { - throw new IOException("Expected integer but got " + e.getMessage()); - } - } - - public IValue visitReal(Type type) throws IOException { - try { - switch (in.peek()) { - case NUMBER: - return vf.real(in.nextDouble()); - case STRING: - return vf.real(in.nextString()); - case NULL: - return null; - default: - throw new IOException("Expected integer but got " + in.peek()); - } - } - catch (NumberFormatException e) { - throw new IOException("Expected integer but got " + e.getMessage()); - } - } - - @Override - public IValue visitExternal(Type type) throws IOException { - throw new IOException("External type " + type + "is not implemented yet by the json reader:" + in.getPath()); - } - - @Override - public IValue visitString(Type type) throws IOException { - if (isNull()) { - return null; - } - - return vf.string(in.nextString()); - } - @Override public IValue visitTuple(Type type) throws IOException { if (isNull()) { @@ -470,6 +335,7 @@ else if (val.contains("://")) { } } + @Override public IValue visitRational(Type type) throws IOException { if (isNull()) { return inferNullValue(nulls, type); @@ -604,6 +470,7 @@ private int getCol() { return 0; } } + protected Throw parseErrorHere(String cause) { var location = src == null ? URIUtil.rootLocation("unknown") : src; int offset = Math.max(getPos(), lastPos); @@ -892,7 +759,7 @@ private boolean isNull() throws IOException { private static final TypeFactory TF = TypeFactory.getInstance(); private final TypeStore store; - private final IRascalValueFactory vf; + private final IValueFactory vf; private ThreadLocal format; private final IRascalMonitor monitor; private ISourceLocation src; @@ -900,10 +767,59 @@ private boolean isNull() throws IOException { private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + private boolean explicitConstructorNames; + private boolean explicitDataTypes; private IFunction parsers; private Map nulls = Collections.emptyMap(); + /** + * @param vf factory which will be used to construct values + * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields + * @param monitor provides progress reports and warnings + * @param src loc to use to identify the entire file. + */ + public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor, ISourceLocation src) { + this.vf = vf; + this.store = store; + this.monitor = monitor; + this.src = src; + setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + + if (src != null) { + try { + var lookup = MethodHandles.lookup(); + var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); + this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); + this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); + this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + this.originTracking = (src != null); + } + catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + // we disable the origin tracking if we can not get to the fields + src = null; + originTracking = false; + monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + } + } + } + public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { + this(vf, new TypeStore(), monitor, src); + } + + public JsonValueReader setExplicitConstructorNames(boolean value) { + this.explicitConstructorNames = value; + return this; + } + + public JsonValueReader setExplicitDataTypes(boolean value) { + this.explicitDataTypes = value; + if (value) { + this.explicitConstructorNames = true; + } + return this; + } + /** * @param vf factory which will be used to construct values * @param store type store to lookup constructors of abstract data-types in and the types of keyword fields @@ -914,7 +830,6 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m this.monitor = monitor; this.src = (src == null) ? URIUtil.rootLocation("unknown") : src; - setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); // this is for origin tracking as well as accurate parse errors @@ -983,4 +898,4 @@ public IValue read(JsonReader in, Type expected) throws IOException { throw dispatch.parseErrorHere(e.getMessage()); } } -} \ No newline at end of file +} From 89ece88cc96c6a29e7d7602f2e28bb7ae5406c3c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 9 Dec 2024 20:33:29 +0100 Subject: [PATCH 35/45] fixed bug in test due to overloading the identifier example --- src/org/rascalmpl/library/lang/json/IO.java | 1 - .../rascal/tests/library/lang/json/JSONIOTests.rsc | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 394db4623cc..43d8fd3ece4 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -142,7 +142,6 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 2e7e13f975e..3f378664094 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -127,12 +127,12 @@ test bool formattingToStringsTest() { } test bool explicitConstructorNames() { - example = data4(e=z()); - json = asJSON(example, explicitConstructorNames=true); + tmp = data4(e=z()); + json = asJSON(tmp, explicitConstructorNames=true); assert json == "{\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\"}}"; - assert parseJSON(#DATA4, json, explicitConstructorNames=true) == example; + assert parseJSON(#DATA4, json, explicitConstructorNames=true) == tmp; // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitConstructorNames=false); @@ -141,17 +141,17 @@ test bool explicitConstructorNames() { } test bool explicitDataTypes() { - example = data4(e=z()); - json = asJSON(example, explicitDataTypes=true); + tmp = data4(e=z()); + json = asJSON(tmp, explicitDataTypes=true); assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; // _constructor and _type must be the first fields - assert parseJSON(#DATA4, json, explicitDataTypes=true) == example; + assert parseJSON(#DATA4, json, explicitDataTypes=true) == tmp; // _type and _constructor may appear in a different order flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; - assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == example; + assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == tmp; // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); From f62cca3805266642d45a3ea10e8556ee038d0797 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 10 Dec 2024 14:23:30 +0100 Subject: [PATCH 36/45] resolved more merge problems --- .../lang/json/internal/JsonValueReader.java | 121 +++++++++++++----- .../tests/library/lang/json/JSONIOTests.rsc | 41 +----- 2 files changed, 96 insertions(+), 66 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index fb7c7b5e436..7f7a09c2b72 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -544,26 +544,84 @@ private IValue visitStringAsAbstractData(Type type) throws IOException { * @throws IOException */ private IValue visitObjectAsAbstractData(Type type) throws IOException { - Set alternatives = store.lookupAlternatives(type); - if (alternatives.size() > 1) { - monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); - } - Type cons = alternatives.iterator().next(); - + Set alternatives = null; + in.beginObject(); int startPos = getPos(); int startLine = getLine(); int startCol = getCol(); + // use explicit information in the JSON to select and filter constructors from the TypeStore + // we expect always to have the field _constructor before _type. + if (explicitConstructorNames || explicitDataTypes) { + String consName = null; + String typeName = null; // this one is optional, and the order with cons is not defined. + String consLabel = in.nextName(); + + // first we read either a cons name or a type name + if (explicitConstructorNames && "_constructor".equals(consLabel)) { + consName = in.nextString(); + } + else if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } + + // optionally read the second field + if (explicitDataTypes && typeName == null) { + // we've read a constructor name, but we still need a type name + consLabel = in.nextName(); + if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } + } + else if (explicitDataTypes && consName == null) { + // we've read type name, but we still need a constructor name + consLabel = in.nextName(); + if (explicitDataTypes && "_constructor".equals(consLabel)) { + consName = in.nextString(); + } + } + + if (explicitDataTypes && typeName == null) { + throw parseErrorHere("Missing a _type field: " + in.getPath()); + } + else if (explicitConstructorNames && consName == null) { + throw parseErrorHere("Missing a _constructor field: " + in.getPath()); + } + + if (typeName != null && consName != null) { + // first focus on the given type name + var dataType = TF.abstractDataType(store, typeName); + alternatives = store.lookupConstructor(dataType, consName); + } + else { + // we only have a constructor name + // lookup over all data types by constructor name + alternatives = store.lookupConstructors(consName); + } + } + else { + alternatives = store.lookupAlternatives(type); + } + + if (alternatives.size() > 1) { + monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); + } + else if (alternatives.size() == 0) { + throw parseErrorHere("No fitting constructor found for " + in.getPath()); + } + + Type cons = alternatives.iterator().next(); + IValue[] args = new IValue[cons.getArity()]; Map kwParams = new HashMap<>(); - + if (!cons.hasFieldNames() && cons.getArity() != 0) { throw parseErrorHere("For the object encoding constructors must have field names " + in.getPath()); } - + while (in.hasNext()) { - String label = nextName(); + String label = in.nextName(); if (cons.hasField(label)) { IValue val = read(in, cons.getFieldType(label)); if (val != null) { @@ -578,29 +636,35 @@ else if (cons.hasKeywordField(label, store)) { IValue val = read(in, store.getKeywordParameterType(cons, label)); // null can still happen if the nulls map doesn't have a default if (val != null) { - // if the value is null we'd use the default value of the defined field in the constructor - kwParams.put(label, val); + // if the value is null we'd use the default value of the defined field in the constructor + kwParams.put(label, val); } } - } - else { // its a normal arg, pass its label to the child - if (!explicitConstructorNames && "_constructor".equals(label)) { - // ignore additional _constructor fields. - in.nextString(); // skip the constructor value - continue; - } - else if (!explicitDataTypes && "_type".equals(label)) { - // ignore additional _type fields. - in.nextString(); // skip the type value - continue; + else { + var nullValue = inferNullValue(nulls, cons.getAbstractDataType()); + if (nullValue != null) { + kwParams.put(label, nullValue); } - else { - // field label does not match data type definition + } + } + else { // its a normal arg, pass its label to the child + if (!explicitConstructorNames && "_constructor".equals(label)) { + // ignore additional _constructor fields. + in.nextString(); // skip the constructor value + continue; + } + else if (!explicitDataTypes && "_type".equals(label)) { + // ignore additional _type fields. + in.nextString(); // skip the type value + continue; + } + else { + // field label does not match data type definition throw parseErrorHere("Unknown field " + label + ":" + in.getPath()); - } + } } - } - + } + in.endObject(); int endPos = getPos(); int endLine = getLine(); @@ -612,11 +676,10 @@ else if (!explicitDataTypes && "_type".equals(label)) { } } - if (originTracking) { + if (src != null) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } - return vf.constructor(cons, args, kwParams); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 3f378664094..f543e5ef0bd 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -91,8 +91,8 @@ test bool dealWithNull() { assert parseJSON(#map[str,Maybe[str]], "{\"bla\": \"foo\"}") == ("bla":just("foo")); // keyword parameters and null - assert parseJSON(#Cons, "{\"bla\": \"foo\"}") == cons(bla="foo"); - assert parseJSON(#Cons, "{\"bla\": null}") == cons(); + assert cons(bla="foo") := parseJSON(#Cons, "{\"bla\": \"foo\"}"); + assert cons() := parseJSON(#Cons, "{\"bla\": null}"); return true; } @@ -147,44 +147,11 @@ test bool explicitDataTypes() { assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; // _constructor and _type must be the first fields - assert parseJSON(#DATA4, json, explicitDataTypes=true) == tmp; + assert tmp := parseJSON(#DATA4, json, explicitDataTypes=true) ; // _type and _constructor may appear in a different order flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; - assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == tmp; - - // here we can't be sure to get z() back, but we will get some Enum - assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); - - return true; -} - -test bool explicitConstructorNames() { - example = data4(e=z()); - json = asJSON(example, explicitConstructorNames=true); - - assert json == "{\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\"}}"; - - assert parseJSON(#DATA4, json, explicitConstructorNames=true) == example; - - // here we can't be sure to get z() back, but we will get some Enum - assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitConstructorNames=false); - - return true; -} - -test bool explicitDataTypes() { - example = data4(e=z()); - json = asJSON(example, explicitDataTypes=true); - - assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; - - // _constructor and _type must be the first fields - assert parseJSON(#DATA4, json, explicitDataTypes=true) == example; - - // _type and _constructor may appear in a different order - flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; - assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == example; + assert tmp := parseJSON(#DATA4, flippedJson, explicitDataTypes=true); // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); From fb3e96fd9a186f80869c095ed706233f2373d823 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 10 Dec 2024 14:48:29 +0100 Subject: [PATCH 37/45] all test work again --- .../lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index f543e5ef0bd..6853dcb52a3 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -132,7 +132,7 @@ test bool explicitConstructorNames() { assert json == "{\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\"}}"; - assert parseJSON(#DATA4, json, explicitConstructorNames=true) == tmp; + assert tmp2 := parseJSON(#DATA4, json, explicitConstructorNames=true) && tmp2 := tmp; // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitConstructorNames=false); @@ -147,11 +147,11 @@ test bool explicitDataTypes() { assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; // _constructor and _type must be the first fields - assert tmp := parseJSON(#DATA4, json, explicitDataTypes=true) ; + assert tmp2 := parseJSON(#DATA4, json, explicitDataTypes=true) && tmp := tmp2 ; // _type and _constructor may appear in a different order flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; - assert tmp := parseJSON(#DATA4, flippedJson, explicitDataTypes=true); + assert tmp2 := parseJSON(#DATA4, flippedJson, explicitDataTypes=true) && tmp := tmp2; // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); From 5adbb8a433f77a74f35c2d1ea5ddeccb2f9d81b4 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 10:53:13 +0100 Subject: [PATCH 38/45] added accurate parse error test --- src/org/rascalmpl/library/lang/json/IO.rsc | 3 ++- .../tests/library/lang/json/JSONIOTests.rsc | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index f0b679a8bc4..2f40b1d4559 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -15,6 +15,7 @@ module lang::json::IO import util::Maybe; +import Exception; @synopsis{JSON parse errors have more information than general parse errors} @description{ @@ -22,7 +23,7 @@ import util::Maybe; * `cause` is a factual diagnosis of what was expected at that position, versus what was found. * `path` is a path query string into the JSON value from the root down to the leaf where the error was detected. } -data RuntimeException = ParseError(loc location, str cause="", str path=""); +data RuntimeException(str cause="", str path=""); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{Maps any Rascal value to a JSON string} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 6853dcb52a3..6f5e55068b5 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -72,6 +72,26 @@ test bool originTracking() { return true; } +test bool accurateParseErrors() { + ex = readFile(|std:///lang/rascal/tests/library/lang/json/glossary.json|); + broken = ex[..size(ex)/2] + ex[size(ex)/2+10..]; + + try { + ex2 = parseJSON(#node, broken, trackOrigins=true); + } + catch ParseError(loc l): + return l.begin.line == 14; + + try { + // accurate locations have to be provided also when trackOrigins=false + ex2 = parseJSON(#node, broken, trackOrigins=false); + } + catch ParseError(loc l): // , cause=/^Unterminated object.*/, path="$.glossary.GlossDiv.GlossList.GlossEntry.GlossTerm") : + return l.begin.line == 14; + + return true; +} + data Cons = cons(str bla = "null"); test bool dealWithNull() { From 3d8a807206bc4e6bb187ef8c4f750f8faf15d3f3 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 14:47:51 +0100 Subject: [PATCH 39/45] made reflective tests thread safe --- .../tests/library/util/ReflectiveTests.rsc | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc index e9df6c2f808..06739ef33d6 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc @@ -1,12 +1,13 @@ module lang::rascal::tests::library::util::ReflectiveTests -import util::Reflective; - -import util::FileSystem; import IO; -import String; import List; +import String; +import util::FileSystem; +import util::Reflective; +import util::UUID; +private loc testLibraryLoc = |memory://myTestLibrary-/|; test bool commonSuffixCommutative(list[str] a, list[str] b) = commonSuffix(a, b) == commonSuffix(b, a); test bool cs1() = commonSuffix([], ["c"]) == 0; @@ -33,35 +34,35 @@ test bool moduleReflectiveWithSrc() { } test bool moduleExceptionOnlyTpl() { - writeFile(|memory://myTestLibrary/resources/rascal/$Exception.tpl|, + writeFile(testLibraryLoc + "/resources/rascal/$Exception.tpl", "$Exception.tpl (only file matters, content irrelevant) "); - pcfg = pathConfig(libs=[|memory://myTestLibrary/resources/|] + pcfg = pathConfig(libs=[testLibraryLoc + "/resources/"] ); return getModuleName(|project://rascal/src/org/rascalmpl/library/Exception.rsc|, pcfg) == "Exception"; } test bool moduleReflectiveOnlyTpl() { - writeFile(|memory://myTestLibrary/resources/rascal/util/Reflective.tpl|, + writeFile(testLibraryLoc + "/resources/rascal/util/Reflective.tpl", "util::$Reflective.tpl (only file matters, content irrelevant) "); pcfg = pathConfig(srcs = [], - libs=[|memory://myTestLibrary/resources/|] + libs=[testLibraryLoc + "/resources/"] ); return getModuleName(|project://rascal/src/org/rascalmpl/library/util/Reflective.rsc|, pcfg) == "util::Reflective"; } test bool longestModuleReflectiveOnlyTpl() { - writeFile(|memory://myTestLibrary1/resources/rascal/$Reflective.tpl|, + writeFile(testLibraryLoc + "/1/resources/rascal/$Reflective.tpl", "$Reflective.tpl at top level (only file matters, content irrelevant) "); - writeFile(|memory://myTestLibrary2/resources/rascal/util/Reflective.tpl|, + writeFile(testLibraryLoc + "/2/resources/rascal/util/Reflective.tpl", "util::$Reflective.tpl in subdir util (only file matters, content irrelevant) "); pcfg = pathConfig(srcs= [], - libs=[|memory://myTestLibrary1/resources/|, |memory://myTestLibrary2/resources/|] + libs=[testLibraryLoc + "1/resources/", testLibraryLoc + "/2/resources/"] ); return getModuleName(|project://rascal/src/org/rascalmpl/library/util/Reflective.rsc|, pcfg) == "util::Reflective"; From d51c5598b40497624ecef5d2fdf2c8dc1e7d57eb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 15:20:41 +0100 Subject: [PATCH 40/45] addresses a part of #2098 by recovering positional parameters for nodes based on the naming scheme arg --- .../lang/json/internal/JsonValueReader.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 7f7a09c2b72..872dafc0402 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -725,13 +725,25 @@ public IValue visitNode(Type type) throws IOException { int startCol = getCol(); Map kws = new HashMap<>(); - + Map args = new HashMap<>(); + while (in.hasNext()) { String kwName = nextName(); - IValue value = read(in, TF.valueType()); - - if (value != null) { - kws.put(kwName, value); + boolean positioned = kwName.startsWith("arg"); + + if (!isNull()) { // lookahead for null to give default parameters the preference. + IValue val = read(in, TF.valueType()); + + if (val != null) { + // if the value is null we'd use the default value of the defined field in the constructor + (positioned ? args : kws).put(kwName, val); + } + } + else { + var nullValue = inferNullValue(nulls, TF.valueType()); + if (nullValue != null) { + (positioned ? args : kws).put(kwName, nullValue); + } } } @@ -744,7 +756,12 @@ public IValue visitNode(Type type) throws IOException { kws.put(kws.containsKey("src") ? "rascal-src" : "src", vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } - return vf.node("object", new IValue[] { }, kws); + IValue[] argArray = args.entrySet().stream() + .sorted((e, f) -> e.getKey().compareTo(f.getKey())) + .map(e -> e.getValue()) + .toArray(IValue[]::new); + + return vf.node("object", argArray, kws); } @Override From 49d4e22ba2ef8e4c75162269d5e764a87bbef2b8 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 15:35:24 +0100 Subject: [PATCH 41/45] fixes #2098 by cleansing random reals and ints and mapping them into the representable range for JSON --- .../tests/library/lang/json/JSONIOTests.rsc | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 6f5e55068b5..c6f03244f85 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -5,23 +5,16 @@ import lang::json::IO; import util::UUID; import util::Maybe; import IO; +import util::Math; loc targetFile = |memory://test-tmp/test-<"">.json|; -bool jsonFeaturesSupported(value v) { - for (/num r := v, size("") > 10) { - // json can only contain double precision numbers (doubles) - // so let's ignore the cases where we get higher random numbers - return false; - } - - return true; -} - bool writeRead(type[&T] returnType, &T dt) { - if (!jsonFeaturesSupported(dt)) { - return true; + dt = visit (dt) { + case real r => fitDouble(r) + case int i => i % floor(pow(2, 10)) when abs(i) > pow(2, 10) } + json = toJSON(dt); return fromJSON(returnType, json) == dt; } From 1800e0c2d0c8c473d51310e1a39bafcec537ba69 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 15:40:13 +0100 Subject: [PATCH 42/45] added some comments --- .../library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index c6f03244f85..ce28b66894f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -11,7 +11,9 @@ loc targetFile = |memory://test-tmp/test-<"">.json|; bool writeRead(type[&T] returnType, &T dt) { dt = visit (dt) { + // reals must fit in double case real r => fitDouble(r) + // integers must not overflow case int i => i % floor(pow(2, 10)) when abs(i) > pow(2, 10) } From 2e4e9c1f671bfc3cebe3de57364c54c88b8d7ccb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 15:42:09 +0100 Subject: [PATCH 43/45] used the wrong part of the UUID URI --- .../library/lang/rascal/tests/library/util/ReflectiveTests.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc index 06739ef33d6..b10804d0ad0 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/util/ReflectiveTests.rsc @@ -7,7 +7,7 @@ import util::FileSystem; import util::Reflective; import util::UUID; -private loc testLibraryLoc = |memory://myTestLibrary-/|; +private loc testLibraryLoc = |memory://myTestLibrary-/|; test bool commonSuffixCommutative(list[str] a, list[str] b) = commonSuffix(a, b) == commonSuffix(b, a); test bool cs1() = commonSuffix([], ["c"]) == 0; From 785a8b7715e646c1ddf91c3514f367905ec6b264 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Dec 2024 15:47:54 +0100 Subject: [PATCH 44/45] set errorsAsWarnings to false for tutor compilation --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b19c32f5c13..c27b3e181cd 100644 --- a/pom.xml +++ b/pom.xml @@ -178,7 +178,7 @@ false false - true + false ${project.build.outputDirectory} ${project.basedir}/LICENSE |http://github.com/usethesource/rascal/blob/main| From 9380bf80bcf459ae663295347a70d7f8a133242d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 18 Dec 2024 09:41:44 +0100 Subject: [PATCH 45/45] commented out the tutor because of changed method signatures in lang::json::IO --- pom.xml | 5 +++-- src/org/rascalmpl/library/lang/html5/DOM.rsc | 6 +----- .../library/lang/rascal/vis/ImportGraph.rsc | 3 ++- src/org/rascalmpl/library/vis/Graphs.rsc | 21 +++++++++++++++++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index c27b3e181cd..b664674a733 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,8 @@ package - + diff --git a/src/org/rascalmpl/library/lang/html5/DOM.rsc b/src/org/rascalmpl/library/lang/html5/DOM.rsc index f8bfd0cbd2d..79127968a1d 100644 --- a/src/org/rascalmpl/library/lang/html5/DOM.rsc +++ b/src/org/rascalmpl/library/lang/html5/DOM.rsc @@ -3,7 +3,6 @@ module lang::html5::DOM import List; -import Content; @synopsis{Generic representation for all HTML tag types.} @description{ @@ -459,7 +458,4 @@ str toString(HTML5Node x) { attrs = { k | HTML5Attr k <- x.kids }; kids = [ k | value k <- x.kids, !(HTML5Attr _ := k) ]; return nodeToString(x.name, attrs, kids); -} - -@synopsis{convenience function to render the HTML5Node dom tree in the browser} -public Content serve(HTML5Node x) = html(toString(x)); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 07e0939441c..4dd099d8eb0 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -107,7 +107,8 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { title="Rascal Import/Extend Graph", nodeClassifier=nodeClass, edgeClassifier=edgeClass, - nodeLinker=modLinker + nodeLinker=modLinker, + edgeStyle=defaultEdgeStyle()[\curve-style=taxi()] ); showInteractiveContent(graph(g, cfg=cfg), title=cfg.title); diff --git a/src/org/rascalmpl/library/vis/Graphs.rsc b/src/org/rascalmpl/library/vis/Graphs.rsc index 42a16471050..a4cb855dda4 100644 --- a/src/org/rascalmpl/library/vis/Graphs.rsc +++ b/src/org/rascalmpl/library/vis/Graphs.rsc @@ -504,6 +504,27 @@ str formatCytoSelector(greaterEqual(str field, int lim)) = "[ \>= ]" str formatCytoSelector(lessEqual(str field, int lim)) = "[ \<= ]"; str formatCytoSelector(less(str field, int lim)) = "[ \< ]"; +@synopsis{Choice of different node layout algorithms.} +@description{ +The different algorithms use heuristics to find a layout +that shows the structure of a graph best. Different types +of graph data call for different algorithms: +* `grid` is best when there are very few edges or when edges are not important. The edge relation +is not used at all for deciding where each node will end up. Grid +is typically used for an initial exploration of the graph. It is very fast. +* `circle` puts all nodes on the edge of a circle and draws edges between them. The order on the +circle is arbitrary. This layout fails on larger collections of nodes because the points on the +circle will become really small and indistinguishable. However for graphs with less than 100 nodes +it provides a quick and natural overview. +* `breadthfirst` computes a breadthfirst spanning tree, and uses path length to decide on which +layer each node will reside. Cross-edges (between branches) and back-edges are allowed but if there +are many the graph will be messy. So this layout is best when you have a mostly hierarchical graph. +Examples are flow charts and dependency graphs. +* `cose` is a so-called "force-directed" layout. The edges become springs that both push nodes +apart as well as pull them together. Nodes drag on the surface but have an initial momentum such +that they can find a spot on the plain. This layout is very natural for scale-free networks such +as biological organisms, friends graphs and software ecosystems. +} data CytoLayoutName = grid() | circle()