From fdc324a4952eb216d69bdecab7d3cbef098207f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 3 Oct 2023 16:06:05 +0200 Subject: [PATCH] Generate doc for assets (#517) --- .github/workflows/ci.yml | 2 +- dev/prepare-minikube-for-e2e-tests.sh | 2 +- .../api/doc/ApiConfigurationModel.java | 3 +- .../ai/langstream/api/doc/AssetConfig.java | 29 ++++ .../api/doc/AssetConfigurationModel.java | 31 ++++ .../api/runtime/AssetNodeProvider.java | 6 + .../api/runtime/PluginsRegistry.java | 5 + .../impl/assets/CassandraAssetsProvider.java | 161 ++++++++++++++++-- .../impl/assets/JdbcAssetsProvider.java | 64 +++++-- .../impl/assets/MilvusAssetsProvider.java | 75 +++++--- .../impl/common/AbstractAssetProvider.java | 118 ++++++++----- .../BaseDataSourceResourceProvider.java | 13 +- .../datasource/AstraDatasourceConfig.java | 14 ++ .../impl/uti/ClassConfigValidator.java | 49 ++++++ .../doc/DocumentationGenerator.java | 11 +- .../doc/DocumentationGeneratorStarter.java | 6 +- 16 files changed, 489 insertions(+), 100 deletions(-) create mode 100644 langstream-api/src/main/java/ai/langstream/api/doc/AssetConfig.java create mode 100644 langstream-api/src/main/java/ai/langstream/api/doc/AssetConfigurationModel.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1435b4bfb..f51973e7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: if: ${{ matrix.name == 'Style' }} run: | #!/bin/bash - mvn clean install -DskipTests -PskipPython + mvn clean install -DskipTests -PskipPython,generateDoc if [[ `git status --porcelain` ]]; then echo "Found changes after building, please re-build the CRDs or run ./mvnw spotless:apply and commit the changes. " git status diff --git a/dev/prepare-minikube-for-e2e-tests.sh b/dev/prepare-minikube-for-e2e-tests.sh index 02dc9a2f0..1b0b5c02f 100755 --- a/dev/prepare-minikube-for-e2e-tests.sh +++ b/dev/prepare-minikube-for-e2e-tests.sh @@ -1,5 +1,4 @@ # -# # Copyright DataStax, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +14,7 @@ # limitations under the License. # +set -e minikube start --memory=8192 --cpus=4 --kubernetes-version=v1.26.3 eval $(minikube docker-env) ./docker/build.sh diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java index d2d534eb4..1495eb3b9 100644 --- a/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java +++ b/langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java @@ -20,4 +20,5 @@ public record ApiConfigurationModel( String version, Map agents, - Map resources) {} + Map resources, + Map assets) {} diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfig.java b/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfig.java new file mode 100644 index 000000000..52569fc99 --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.doc; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface AssetConfig { + String name() default ""; + + String description() default ""; +} diff --git a/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfigurationModel.java b/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfigurationModel.java new file mode 100644 index 000000000..5100bf5ef --- /dev/null +++ b/langstream-api/src/main/java/ai/langstream/api/doc/AssetConfigurationModel.java @@ -0,0 +1,31 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.api.doc; + +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AssetConfigurationModel { + + private String name; + private String description; + private Map properties; +} diff --git a/langstream-api/src/main/java/ai/langstream/api/runtime/AssetNodeProvider.java b/langstream-api/src/main/java/ai/langstream/api/runtime/AssetNodeProvider.java index 5f8181c9c..a440f643b 100644 --- a/langstream-api/src/main/java/ai/langstream/api/runtime/AssetNodeProvider.java +++ b/langstream-api/src/main/java/ai/langstream/api/runtime/AssetNodeProvider.java @@ -15,8 +15,10 @@ */ package ai.langstream.api.runtime; +import ai.langstream.api.doc.AssetConfigurationModel; import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.model.Module; +import java.util.Map; public interface AssetNodeProvider { @@ -45,4 +47,8 @@ AssetNode createImplementation( * @return true if this provider can create the implementation */ boolean supports(String type, ComputeClusterRuntime clusterRuntime); + + default Map generateSupportedTypesDocumentation() { + return Map.of(); + } } diff --git a/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java b/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java index d51397e49..8121b3b53 100644 --- a/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java +++ b/langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java @@ -52,6 +52,11 @@ public List lookupAvailableAgentImplementations() { return loader.stream().map(p -> p.get()).collect(Collectors.toList()); } + public List lookupAvailableAssetImplementations() { + ServiceLoader loader = ServiceLoader.load(AssetNodeProvider.class); + return loader.stream().map(p -> p.get()).collect(Collectors.toList()); + } + public AssetNodeProvider lookupAssetImplementation( String type, ComputeClusterRuntime clusterRuntime) { log.info( diff --git a/langstream-core/src/main/java/ai/langstream/impl/assets/CassandraAssetsProvider.java b/langstream-core/src/main/java/ai/langstream/impl/assets/CassandraAssetsProvider.java index de6269577..de34e8195 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/assets/CassandraAssetsProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/assets/CassandraAssetsProvider.java @@ -15,50 +15,61 @@ */ package ai.langstream.impl.assets; -import static ai.langstream.api.util.ConfigurationUtils.requiredField; import static ai.langstream.api.util.ConfigurationUtils.requiredListField; import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; +import ai.langstream.api.doc.AssetConfig; +import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.util.ConfigurationUtils; import ai.langstream.impl.common.AbstractAssetProvider; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; import java.util.Set; +import lombok.Data; import lombok.extern.slf4j.Slf4j; @Slf4j public class CassandraAssetsProvider extends AbstractAssetProvider { + protected static final String CASSANDRA_TABLE = "cassandra-table"; + protected static final String CASSANDRA_KEYSPACE = "cassandra-keyspace"; + protected static final String ASTRA_KEYSPACE = "astra-keyspace"; + public CassandraAssetsProvider() { - super(Set.of("cassandra-table", "cassandra-keyspace", "astra-keyspace")); + super(Set.of(CASSANDRA_TABLE, CASSANDRA_KEYSPACE, ASTRA_KEYSPACE)); + } + + @Override + protected Class getAssetConfigModelClass(String type) { + return switch (type) { + case CASSANDRA_TABLE -> CassandraTableConfig.class; + case CASSANDRA_KEYSPACE -> CassandraKeyspaceConfig.class; + case ASTRA_KEYSPACE -> AstraKeyspaceConfig.class; + default -> throw new IllegalArgumentException("Unknown asset type " + type); + }; } @Override protected void validateAsset(AssetDefinition assetDefinition, Map asset) { Map configuration = ConfigurationUtils.getMap("config", null, asset); - requiredField(configuration, "datasource", describe(assetDefinition)); final Map datasource = ConfigurationUtils.getMap("datasource", Map.of(), configuration); final Map datasourceConfiguration = ConfigurationUtils.getMap("configuration", Map.of(), datasource); switch (assetDefinition.getAssetType()) { - case "cassandra-table" -> { - requiredNonEmptyField(configuration, "table-name", describe(assetDefinition)); - requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition)); - requiredListField(configuration, "create-statements", describe(assetDefinition)); + case CASSANDRA_TABLE -> { checkDeleteStatements(assetDefinition, configuration); } - case "cassandra-keyspace" -> { - requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition)); - requiredListField(configuration, "create-statements", describe(assetDefinition)); + case CASSANDRA_KEYSPACE -> { checkDeleteStatements(assetDefinition, configuration); if (datasourceConfiguration.containsKey("secureBundle") || datasourceConfiguration.containsKey("database")) { throw new IllegalArgumentException("Use astra-keyspace for AstraDB services"); } } - case "astra-keyspace" -> { - requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition)); + case ASTRA_KEYSPACE -> { if (!datasourceConfiguration.containsKey("secureBundle") && !datasourceConfiguration.containsKey("database")) { throw new IllegalArgumentException( @@ -70,8 +81,7 @@ protected void validateAsset(AssetDefinition assetDefinition, Map throw new IllegalStateException( - "Unexpected value: " + assetDefinition.getAssetType()); + default -> {} } } @@ -89,6 +99,127 @@ private static void checkDeleteStatements( @Override protected boolean lookupResource(String fieldName) { - return "datasource".contains(fieldName); + return "datasource".equals(fieldName); + } + + @AssetConfig( + name = "Cassandra table", + description = + """ + Manage a Cassandra table in existing keyspace. + """) + @Data + public static class CassandraTableConfig { + + @ConfigProperty( + description = + """ + Reference to a datasource id configured in the application. + """, + required = true) + private String datasource; + + @ConfigProperty( + description = + """ + Name of the table. + """, + required = true) + @JsonProperty("table-name") + private String table; + + @ConfigProperty( + description = + """ + Name of the keyspace where the table is located. + """, + required = true) + private String keyspace; + + @ConfigProperty( + description = + """ + List of the statement to execute to create the table. They will be executed every time the application is deployed or upgraded. + """, + required = true) + @JsonProperty("create-statements") + private List createStatements; + + @ConfigProperty( + description = + """ + List of the statement to execute to cleanup the table. They will be executed when the application is deleted only if 'deletion-mode' is 'delete'. + """) + @JsonProperty("delete-statements") + private List deleteStatements; + } + + @AssetConfig( + name = "Cassandra keyspace", + description = + """ + Manage a Cassandra keyspace. + """) + @Data + public static class CassandraKeyspaceConfig { + + @ConfigProperty( + description = + """ + Reference to a datasource id configured in the application. + """, + required = true) + private String datasource; + + @ConfigProperty( + description = + """ + Name of the keyspace to create. + """, + required = true) + private String keyspace; + + @ConfigProperty( + description = + """ + List of the statement to execute to create the keyspace. They will be executed every time the application is deployed or upgraded. + """, + required = true) + @JsonProperty("create-statements") + private List createStatements; + + @ConfigProperty( + description = + """ + List of the statement to execute to cleanup the keyspace. They will be executed when the application is deleted only if 'deletion-mode' is 'delete'. + """) + @JsonProperty("delete-statements") + private List deleteStatements; + } + + @AssetConfig( + name = "Astra keyspace", + description = + """ + Manage a DataStax Astra keyspace. + """) + @Data + public static class AstraKeyspaceConfig { + + @ConfigProperty( + description = + """ + Reference to a datasource id configured in the application. + """, + required = true) + private String datasource; + + @ConfigProperty( + description = + """ + Name of the keyspace to create. + """, + required = true) + private String keyspace; } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/assets/JdbcAssetsProvider.java b/langstream-core/src/main/java/ai/langstream/impl/assets/JdbcAssetsProvider.java index 5cc50200c..825125c32 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/assets/JdbcAssetsProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/assets/JdbcAssetsProvider.java @@ -15,15 +15,18 @@ */ package ai.langstream.impl.assets; -import static ai.langstream.api.util.ConfigurationUtils.requiredField; import static ai.langstream.api.util.ConfigurationUtils.requiredListField; -import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; +import ai.langstream.api.doc.AssetConfig; +import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.util.ConfigurationUtils; import ai.langstream.impl.common.AbstractAssetProvider; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; import java.util.Set; +import lombok.Data; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -33,18 +36,16 @@ public JdbcAssetsProvider() { super(Set.of("jdbc-table")); } + @Override + protected Class getAssetConfigModelClass(String type) { + return TableConfig.class; + } + @Override protected void validateAsset(AssetDefinition assetDefinition, Map asset) { Map configuration = ConfigurationUtils.getMap("config", null, asset); - requiredField(configuration, "datasource", describe(assetDefinition)); - final Map datasource = - ConfigurationUtils.getMap("datasource", Map.of(), configuration); - final Map datasourceConfiguration = - ConfigurationUtils.getMap("configuration", Map.of(), datasource); switch (assetDefinition.getAssetType()) { case "jdbc-table" -> { - requiredNonEmptyField(configuration, "table-name", describe(assetDefinition)); - requiredListField(configuration, "create-statements", describe(assetDefinition)); checkDeleteStatements(assetDefinition, configuration); } default -> throw new IllegalStateException( @@ -66,6 +67,49 @@ private static void checkDeleteStatements( @Override protected boolean lookupResource(String fieldName) { - return "datasource".contains(fieldName); + return "datasource".equals(fieldName); + } + + @AssetConfig( + name = "JDBC table", + description = """ + Manage a JDBC table. + """) + @Data + public static class TableConfig { + + @ConfigProperty( + description = + """ + Reference to a datasource id configured in the application. + """, + required = true) + private String datasource; + + @ConfigProperty( + description = + """ + Name of the table. + """, + required = true) + @JsonProperty("table-name") + private String table; + + @ConfigProperty( + description = + """ + List of the statement to execute to create the table. They will be executed every time the application is deployed or upgraded. + """, + required = true) + @JsonProperty("create-statements") + private List createStatements; + + @ConfigProperty( + description = + """ + List of the statement to execute to cleanup the table. They will be executed when the application is deleted only if 'deletion-mode' is 'delete'. + """) + @JsonProperty("delete-statements") + private List deleteStatements; } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/assets/MilvusAssetsProvider.java b/langstream-core/src/main/java/ai/langstream/impl/assets/MilvusAssetsProvider.java index f79f3d7be..e90955cb5 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/assets/MilvusAssetsProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/assets/MilvusAssetsProvider.java @@ -15,15 +15,13 @@ */ package ai.langstream.impl.assets; -import static ai.langstream.api.util.ConfigurationUtils.requiredField; -import static ai.langstream.api.util.ConfigurationUtils.requiredListField; -import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; - -import ai.langstream.api.model.AssetDefinition; -import ai.langstream.api.util.ConfigurationUtils; +import ai.langstream.api.doc.AssetConfig; +import ai.langstream.api.doc.ConfigProperty; import ai.langstream.impl.common.AbstractAssetProvider; -import java.util.Map; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Set; +import lombok.Data; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -34,25 +32,56 @@ public MilvusAssetsProvider() { } @Override - protected void validateAsset(AssetDefinition assetDefinition, Map asset) { - Map configuration = ConfigurationUtils.getMap("config", null, asset); - requiredField(configuration, "datasource", describe(assetDefinition)); - final Map datasource = - ConfigurationUtils.getMap("datasource", Map.of(), configuration); - final Map datasourceConfiguration = - ConfigurationUtils.getMap("configuration", Map.of(), datasource); - switch (assetDefinition.getAssetType()) { - case "milvus-collection" -> { - requiredNonEmptyField(configuration, "collection-name", describe(assetDefinition)); - requiredListField(configuration, "create-statements", describe(assetDefinition)); - } - default -> throw new IllegalStateException( - "Unexpected value: " + assetDefinition.getAssetType()); - } + protected Class getAssetConfigModelClass(String type) { + return TableConfig.class; } @Override protected boolean lookupResource(String fieldName) { - return "datasource".contains(fieldName); + return "datasource".equals(fieldName); + } + + @AssetConfig( + name = "Milvus collection", + description = + """ + Manage a Milvus collection. + """) + @Data + public static class TableConfig { + + @ConfigProperty( + description = + """ + Reference to a datasource id configured in the application. + """, + required = true) + private String datasource; + + @ConfigProperty( + description = + """ + Name of the collection. + """, + required = true) + @JsonProperty("collection-name") + private String collectionName; + + @ConfigProperty( + description = + """ + Name of the database where to create the collection. + """) + @JsonProperty("database-name") + private String databaseName; + + @ConfigProperty( + description = + """ + List of the statement to execute to create the collection. They will be executed every time the application is deployed or upgraded. + """, + required = true) + @JsonProperty("create-statements") + private List createStatements; } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/common/AbstractAssetProvider.java b/langstream-core/src/main/java/ai/langstream/impl/common/AbstractAssetProvider.java index 26e6210c4..1f50a07e1 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/common/AbstractAssetProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/common/AbstractAssetProvider.java @@ -17,6 +17,7 @@ import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField; +import ai.langstream.api.doc.AssetConfigurationModel; import ai.langstream.api.model.Application; import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.model.Module; @@ -26,7 +27,9 @@ import ai.langstream.api.runtime.ComputeClusterRuntime; import ai.langstream.api.runtime.ExecutionPlan; import ai.langstream.api.runtime.PluginsRegistry; +import ai.langstream.impl.uti.ClassConfigValidator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -34,10 +37,10 @@ /** Utility method to implement an AssetNodeProvider. */ public abstract class AbstractAssetProvider implements AssetNodeProvider { - private final Set supportedType; + private final Set supportedTypes; - public AbstractAssetProvider(Set supportedType) { - this.supportedType = supportedType; + public AbstractAssetProvider(Set supportedTypes) { + this.supportedTypes = supportedTypes; } @Override @@ -57,8 +60,8 @@ public final AssetNode createImplementation( return new AssetNode(asset); } - protected abstract void validateAsset( - AssetDefinition assetDefinition, Map asset); + protected void validateAsset(AssetDefinition assetDefinition, Map asset) {} + ; private Map planAsset( AssetDefinition assetDefinition, @@ -66,48 +69,53 @@ private Map planAsset( ComputeClusterRuntime computeClusterRuntime, PluginsRegistry pluginsRegistry) { - if (!supportedType.contains(assetDefinition.getAssetType())) { + final String type = assetDefinition.getAssetType(); + if (!supportedTypes.contains(type)) { throw new IllegalStateException(); } + + final Class assetConfigModelClass = getAssetConfigModelClass(type); + final Map config = assetDefinition.getConfig(); + if (assetConfigModelClass != null) { + ClassConfigValidator.validateAssetModelFromClass( + assetDefinition, + getAssetConfigModelClass(type), + config, + isAssetConfigModelAllowUnknownProperties(type)); + } Map resources = application.getResources(); Map asset = new HashMap<>(); asset.put("id", assetDefinition.getId()); asset.put("name", assetDefinition.getName()); - asset.put("asset-type", assetDefinition.getAssetType()); + asset.put("asset-type", type); asset.put("creation-mode", assetDefinition.getCreationMode()); asset.put("deletion-mode", assetDefinition.getDeletionMode()); Map configuration = new HashMap<>(); - if (assetDefinition.getConfig() != null) { - assetDefinition - .getConfig() - .forEach( - (key, value) -> { - // automatically resolve resource references - // should we do it depending on the asset type ? - if (lookupResource(key)) { - String resourceId = - requiredNonEmptyField( - assetDefinition.getConfig(), - key, - describe(assetDefinition)); - Resource resource = resources.get(resourceId); - if (resource != null) { - Map resourceImplementation = - computeClusterRuntime.getResourceImplementation( - resource, pluginsRegistry); - value = Map.of("configuration", resourceImplementation); - } else { - throw new IllegalArgumentException( - "Resource with name=" - + resourceId - + " not found, declared as " - + key - + " in asset " - + assetDefinition.getId()); - } - } - configuration.put(key, value); - }); + if (config != null) { + config.forEach( + (key, value) -> { + // automatically resolve resource references + if (lookupResource(key)) { + String resourceId = + requiredNonEmptyField(config, key, describe(assetDefinition)); + Resource resource = resources.get(resourceId); + if (resource != null) { + Map resourceImplementation = + computeClusterRuntime.getResourceImplementation( + resource, pluginsRegistry); + value = Map.of("configuration", resourceImplementation); + } else { + throw new IllegalArgumentException( + "Resource with id=" + + resourceId + + " not found, declared as " + + key + + " in asset " + + assetDefinition.getId()); + } + } + configuration.put(key, value); + }); } asset.put("config", configuration); return asset; @@ -115,18 +123,36 @@ private Map planAsset( protected abstract boolean lookupResource(String fieldName); + protected Class getAssetConfigModelClass(String type) { + return null; + } + + protected boolean isAssetConfigModelAllowUnknownProperties(String type) { + return false; + } + @Override public boolean supports(String type, ComputeClusterRuntime clusterRuntime) { - return supportedType.contains(type); + return supportedTypes.contains(type); } protected static Supplier describe(AssetDefinition assetDefinition) { - return () -> - "asset definition, type=" - + assetDefinition.getAssetType() - + ", name=" - + assetDefinition.getName() - + ", id=" - + assetDefinition.getId(); + return () -> new ClassConfigValidator.AssetEntityRef(assetDefinition).ref(); + } + + @Override + public Map generateSupportedTypesDocumentation() { + Map result = new LinkedHashMap<>(); + for (String supportedType : supportedTypes) { + final Class modelClass = getAssetConfigModelClass(supportedType); + if (modelClass == null) { + result.put(supportedType, new AssetConfigurationModel()); + } else { + result.put( + supportedType, + ClassConfigValidator.generateAssetModelFromClass(modelClass)); + } + } + return result; } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java index 166f9f6c3..2427d142a 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/BaseDataSourceResourceProvider.java @@ -23,13 +23,16 @@ import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.api.runtime.ResourceNodeProvider; import ai.langstream.impl.uti.ClassConfigValidator; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.LinkedHashMap; import java.util.Map; import lombok.AllArgsConstructor; +import lombok.SneakyThrows; @AllArgsConstructor public class BaseDataSourceResourceProvider implements ResourceNodeProvider { + protected static final ObjectMapper MAPPER = new ObjectMapper(); private final String resourceType; private final Map supportedServices; @@ -73,9 +76,11 @@ public Map generateSupportedTypesDocumentati Map result = new LinkedHashMap<>(); for (Map.Entry datasource : supportedServices.entrySet()) { final String service = datasource.getKey(); - final ResourceConfigurationModel value = + ResourceConfigurationModel value = ClassConfigValidator.generateResourceModelFromClass( datasource.getValue().getResourceConfigModelClass()); + value = deepCopy(value); + value.getProperties() .get("service") .setDescription("Service type. Set to '" + service + "'"); @@ -85,4 +90,10 @@ public Map generateSupportedTypesDocumentati } return result; } + + @SneakyThrows + private static ResourceConfigurationModel deepCopy(ResourceConfigurationModel instance) { + return MAPPER.readValue( + MAPPER.writeValueAsBytes(instance), ResourceConfigurationModel.class); + } } diff --git a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java index 69bcf6dbc..b144e4190 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java +++ b/langstream-core/src/main/java/ai/langstream/impl/resources/datasource/AstraDatasourceConfig.java @@ -109,6 +109,20 @@ Astra Token (AstraCS:xxx) for connecting to the database. If secureBundle is pro required = true) private String secret; + @ConfigProperty( + description = + """ + DEPRECATED: use clientId instead. + """) + private String username; + + @ConfigProperty( + description = + """ + DEPRECATED: use secret instead. + """) + private String password; + public enum Environments { DEV, PROD, diff --git a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java index b3e9a9738..144d6265e 100644 --- a/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java +++ b/langstream-core/src/main/java/ai/langstream/impl/uti/ClassConfigValidator.java @@ -17,11 +17,14 @@ import ai.langstream.api.doc.AgentConfig; import ai.langstream.api.doc.AgentConfigurationModel; +import ai.langstream.api.doc.AssetConfig; +import ai.langstream.api.doc.AssetConfigurationModel; import ai.langstream.api.doc.ConfigProperty; import ai.langstream.api.doc.ConfigPropertyIgnore; import ai.langstream.api.doc.ResourceConfig; import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.model.AgentConfiguration; +import ai.langstream.api.model.AssetDefinition; import ai.langstream.api.model.Resource; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -60,6 +63,7 @@ public class ClassConfigValidator { static final Map agentModels = new ConcurrentHashMap<>(); static final Map resourceModels = new ConcurrentHashMap<>(); + static final Map assetModels = new ConcurrentHashMap<>(); public static AgentConfigurationModel generateAgentModelFromClass(Class clazz) { return agentModels.computeIfAbsent( @@ -104,6 +108,27 @@ private static ResourceConfigurationModel generateResourceModelFromClassNoCache( return model; } + public static AssetConfigurationModel generateAssetModelFromClass(Class clazz) { + return assetModels.computeIfAbsent( + clazz.getName(), k -> generateAssetFromClassNoCache(clazz)); + } + + private static AssetConfigurationModel generateAssetFromClassNoCache(Class clazz) { + AssetConfigurationModel model = new AssetConfigurationModel(); + + final AssetConfig assetConfig = (AssetConfig) clazz.getAnnotation(AssetConfig.class); + if (assetConfig != null) { + if (assetConfig.description() != null && !assetConfig.description().isBlank()) { + model.setDescription(assetConfig.description().strip()); + } + if (assetConfig.name() != null && !assetConfig.name().isBlank()) { + model.setName(assetConfig.name()); + } + } + model.setProperties(readPropertiesFromClass(clazz)); + return model; + } + public interface EntityRef { String ref(); } @@ -155,6 +180,30 @@ public static void validateResourceModelFromClass( new ResourceEntityRef(resource), modelClazz, asMap, allowUnknownProperties); } + @AllArgsConstructor + public static class AssetEntityRef implements EntityRef { + + private final AssetDefinition asset; + + @Override + public String ref() { + return "asset configuration (asset: '%s', type: '%s')" + .formatted( + asset.getName() == null ? asset.getId() : asset.getName(), + asset.getAssetType()); + } + } + + @SneakyThrows + public static void validateAssetModelFromClass( + AssetDefinition asset, + Class modelClazz, + Map asMap, + boolean allowUnknownProperties) { + validateModelFromClass( + new AssetEntityRef(asset), modelClazz, asMap, allowUnknownProperties); + } + @SneakyThrows private static void validateModelFromClass( EntityRef entityRef, diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java index 3882b7371..b4bbe4d9c 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGenerator.java @@ -17,8 +17,10 @@ import ai.langstream.api.doc.AgentConfigurationModel; import ai.langstream.api.doc.ApiConfigurationModel; +import ai.langstream.api.doc.AssetConfigurationModel; import ai.langstream.api.doc.ResourceConfigurationModel; import ai.langstream.api.runtime.AgentNodeProvider; +import ai.langstream.api.runtime.AssetNodeProvider; import ai.langstream.api.runtime.PluginsRegistry; import ai.langstream.api.runtime.ResourceNodeProvider; import java.util.List; @@ -32,6 +34,7 @@ public class DocumentationGenerator { public static ApiConfigurationModel generateDocs(String version) { Map agents = new TreeMap<>(); Map resources = new TreeMap<>(); + Map assets = new TreeMap<>(); final PluginsRegistry registry = new PluginsRegistry(); final List nodes = registry.lookupAvailableAgentImplementations(); for (AgentNodeProvider node : nodes) { @@ -43,6 +46,12 @@ public static ApiConfigurationModel generateDocs(String version) { for (ResourceNodeProvider resourceNodeProvider : resourceNodeProviders) { resources.putAll(resourceNodeProvider.generateSupportedTypesDocumentation()); } - return new ApiConfigurationModel(version, agents, resources); + + final List assetNodeProviders = + registry.lookupAvailableAssetImplementations(); + for (AssetNodeProvider assetProvider : assetNodeProviders) { + assets.putAll(assetProvider.generateSupportedTypesDocumentation()); + } + return new ApiConfigurationModel(version, agents, resources, assets); } } diff --git a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java index aaead2960..2b8bf5788 100644 --- a/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java +++ b/langstream-webservice/src/main/java/ai/langstream/webservice/doc/DocumentationGeneratorStarter.java @@ -45,7 +45,11 @@ public static void main(String[] args) { final ApiConfigurationModel model = DocumentationGenerator.generateDocs(version); jsonWriter.writeValue(agentsFile.toFile(), model); System.out.println( - "Generated documentation with %d agents".formatted(model.agents().size())); + "Generated documentation with %d agents, %d resources, %d assets" + .formatted( + model.agents().size(), + model.resources().size(), + model.assets().size())); } catch (Throwable e) { e.printStackTrace();