From 4bdeaa6fd249bdd574e4a62eb334f0b09955b966 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Mon, 2 Oct 2023 21:50:47 +0200 Subject: [PATCH] [compatibility] Backward compatibility with references to secrets and globals using the Mustache syntax (#515) --- .../ApplicationPlaceholderResolver.java | 41 +++++- .../ApplicationPlaceholderResolverTest.java | 130 ++++++++++++++++++ 2 files changed, 166 insertions(+), 5 deletions(-) diff --git a/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java b/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java index 1d46f52d3..d7ae6b232 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java +++ b/langstream-core/src/main/java/ai/langstream/impl/common/ApplicationPlaceholderResolver.java @@ -36,6 +36,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -272,20 +273,50 @@ static Object resolveSingleValue(Map context, String template) { return resolvePlaceholdersInString(template, context); } + private static Set ONLY_SECRETS_AND_GLOBALS = Set.of("secrets", "globals"); + static String resolvePlaceholdersInString(String template, Map context) { + if (!template.contains("${") && template.contains("{{") && template.contains("}}")) { + // 0.x syntax + if (template.contains("{{{")) { + return resolvePlaceholdersInString( + template, context, "{{{", "}}}", ONLY_SECRETS_AND_GLOBALS); + } else { + return resolvePlaceholdersInString( + template, context, "{{", "}}", ONLY_SECRETS_AND_GLOBALS); + } + } + return resolvePlaceholdersInString(template, context, "${", "}", null); + } + + private static String resolvePlaceholdersInString( + String template, + Map context, + String prefix, + String suffix, + Set allowedRoots) { StringBuilder result = new StringBuilder(); int position = 0; - int pos = template.indexOf("${", position); + int pos = template.indexOf(prefix, position); if (pos < 0) { return template; } while (pos >= 0) { result.append(template, position, pos); - int end = template.indexOf("}", pos); + int end = template.indexOf(suffix, pos); if (end < 0) { throw new IllegalArgumentException("Invalid placeholder: " + template); } - String placeholder = template.substring(pos + 2, end).trim(); + String placeholder = template.substring(pos + prefix.length(), end).trim(); + + if (allowedRoots != null && allowedRoots.stream().noneMatch(placeholder::startsWith)) { + // this is a raw value like "The question is {{{value.question}}}" + // in a mustache template + // at this step of the preprocessor we allow mustache like syntax only for secrets + // and globals + // in order to keep compatibility with 0.x applications + return template; + } Object value = resolveReference(placeholder, context); if (value == null) { // to not write "null" inside the string @@ -301,8 +332,8 @@ static String resolvePlaceholdersInString(String template, Map c } } result.append(value); - position = end + 1; - pos = template.indexOf("${", position); + position = end + suffix.length(); + pos = template.indexOf(prefix, position); } result.append(template, position, template.length()); return result.toString(); diff --git a/langstream-core/src/test/java/ai/langstream/impl/common/ApplicationPlaceholderResolverTest.java b/langstream-core/src/test/java/ai/langstream/impl/common/ApplicationPlaceholderResolverTest.java index 5d2636086..e4ba14a37 100644 --- a/langstream-core/src/test/java/ai/langstream/impl/common/ApplicationPlaceholderResolverTest.java +++ b/langstream-core/src/test/java/ai/langstream/impl/common/ApplicationPlaceholderResolverTest.java @@ -424,4 +424,134 @@ void testResolve() { ApplicationPlaceholderResolver.resolveSingleValue( context, "${ globals.foo.number }-${ globals.foo.map }")); } + + @Test + void testResolveCompatibilityTripleBraces() { + Map context = + Map.of( + "globals", + Map.of( + "foo", + Map.of( + "bar", + "xxx", + "number", + 123, + "list", + List.of(1, 2), + "map", + Map.of("one", 1, "two", 2)))); + assertEquals( + "xxx", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{globals.foo.bar}}}")); + assertEquals( + "123", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{globals.foo.number}}}")); + assertEquals( + "[1,2]", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{globals.foo.list}}}")); + + // some spaces + assertEquals( + "123", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{ globals.foo.number }}}")); + + // simple concat + assertEquals( + "123-xxx", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{ globals.foo.number }}}-{{{ globals.foo.bar }}}")); + + // using a list, but in a string context + assertEquals( + "123-[1,2]", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{ globals.foo.number }}}-{{{ globals.foo.list }}}")); + + // using a map, but in a string context + assertEquals( + "123-{\"one\":1,\"two\":2}", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{{ globals.foo.number }}}-{{{ globals.foo.map }}}")); + } + + @Test + void testResolveCompatibilityDoubleBraces() { + Map context = + Map.of( + "globals", + Map.of( + "foo", + Map.of( + "bar", + "xxx", + "number", + 123, + "list", + List.of(1, 2), + "map", + Map.of("one", 1, "two", 2)))); + assertEquals( + "xxx", + ApplicationPlaceholderResolver.resolveSingleValue(context, "{{globals.foo.bar}}")); + assertEquals( + "123", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{globals.foo.number}}")); + assertEquals( + "[1,2]", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue(context, "{{globals.foo.list}}")); + + // some spaces + assertEquals( + "123", // this is a string ! + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{ globals.foo.number }}")); + + // simple concat + assertEquals( + "123-xxx", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{ globals.foo.number }}-{{ globals.foo.bar }}")); + + // using a list, but in a string context + assertEquals( + "123-[1,2]", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{ globals.foo.number }}-{{ globals.foo.list }}")); + + // using a map, but in a string context + assertEquals( + "123-{\"one\":1,\"two\":2}", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{ globals.foo.number }}-{{ globals.foo.map }}")); + } + + @Test + void testDontBreakAMustacheValue() { + Map context = + Map.of( + "something", + Map.of("foo", Map.of("bar", "xxx", "number", 123, "list", List.of(1, 2))), + "globals", + Map.of( + "foo", + Map.of( + "bar", + "xxx", + "number", + 123, + "list", + List.of(1, 2), + "map", + Map.of("one", 1, "two", 2)))); + assertEquals( + "{{something.foo.bar}}", + ApplicationPlaceholderResolver.resolveSingleValue( + context, "{{something.foo.bar}}")); + } }