Generate doc for assets (#517)

.github/workflows/ci.yml

@@ -109,7 +109,7 @@ jobs:
         if: ${{ matrix.name == 'Style' }}
         run: |
           #!/bin/bash
-          mvn clean install -DskipTests -PskipPython
+          mvn clean install -DskipTests -PskipPython,generateDoc
           if [[ `git status --porcelain` ]]; then
             echo "Found changes after building, please re-build the CRDs or run ./mvnw spotless:apply and commit the changes. "
             git status

dev/prepare-minikube-for-e2e-tests.sh

@@ -1,5 +1,4 @@
 #
-#
 # Copyright DataStax, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,7 @@
 # limitations under the License.
 #
 
+set -e
 minikube start --memory=8192 --cpus=4 --kubernetes-version=v1.26.3
 eval $(minikube docker-env)
 ./docker/build.sh

langstream-api/src/main/java/ai/langstream/api/doc/ApiConfigurationModel.java

@@ -20,4 +20,5 @@
 public record ApiConfigurationModel(
         String version,
         Map<String, AgentConfigurationModel> agents,
-        Map<String, ResourceConfigurationModel> resources) {}
+        Map<String, ResourceConfigurationModel> resources,
+        Map<String, AssetConfigurationModel> assets) {}

langstream-api/src/main/java/ai/langstream/api/doc/AssetConfig.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.doc;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface AssetConfig {
+    String name() default "";
+
+    String description() default "";
+}

langstream-api/src/main/java/ai/langstream/api/doc/AssetConfigurationModel.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.api.doc;
+
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class AssetConfigurationModel {
+
+    private String name;
+    private String description;
+    private Map<String, ConfigPropertyModel> properties;
+}

langstream-api/src/main/java/ai/langstream/api/runtime/AssetNodeProvider.java

@@ -15,8 +15,10 @@
  */
 package ai.langstream.api.runtime;
 
+import ai.langstream.api.doc.AssetConfigurationModel;
 import ai.langstream.api.model.AssetDefinition;
 import ai.langstream.api.model.Module;
+import java.util.Map;
 
 public interface AssetNodeProvider {
 
@@ -45,4 +47,8 @@ AssetNode createImplementation(
      * @return true if this provider can create the implementation
      */
     boolean supports(String type, ComputeClusterRuntime clusterRuntime);
+
+    default Map<String, AssetConfigurationModel> generateSupportedTypesDocumentation() {
+        return Map.of();
+    }
 }

langstream-api/src/main/java/ai/langstream/api/runtime/PluginsRegistry.java

@@ -52,6 +52,11 @@ public List<AgentNodeProvider> lookupAvailableAgentImplementations() {
         return loader.stream().map(p -> p.get()).collect(Collectors.toList());
     }
 
+    public List<AssetNodeProvider> lookupAvailableAssetImplementations() {
+        ServiceLoader<AssetNodeProvider> loader = ServiceLoader.load(AssetNodeProvider.class);
+        return loader.stream().map(p -> p.get()).collect(Collectors.toList());
+    }
+
     public AssetNodeProvider lookupAssetImplementation(
             String type, ComputeClusterRuntime clusterRuntime) {
         log.info(

langstream-core/src/main/java/ai/langstream/impl/assets/CassandraAssetsProvider.java

@@ -15,50 +15,61 @@
  */
 package ai.langstream.impl.assets;
 
-import static ai.langstream.api.util.ConfigurationUtils.requiredField;
 import static ai.langstream.api.util.ConfigurationUtils.requiredListField;
 import static ai.langstream.api.util.ConfigurationUtils.requiredNonEmptyField;
 
+import ai.langstream.api.doc.AssetConfig;
+import ai.langstream.api.doc.ConfigProperty;
 import ai.langstream.api.model.AssetDefinition;
 import ai.langstream.api.util.ConfigurationUtils;
 import ai.langstream.impl.common.AbstractAssetProvider;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class CassandraAssetsProvider extends AbstractAssetProvider {
 
+    protected static final String CASSANDRA_TABLE = "cassandra-table";
+    protected static final String CASSANDRA_KEYSPACE = "cassandra-keyspace";
+    protected static final String ASTRA_KEYSPACE = "astra-keyspace";
+
     public CassandraAssetsProvider() {
-        super(Set.of("cassandra-table", "cassandra-keyspace", "astra-keyspace"));
+        super(Set.of(CASSANDRA_TABLE, CASSANDRA_KEYSPACE, ASTRA_KEYSPACE));
+    }
+
+    @Override
+    protected Class getAssetConfigModelClass(String type) {
+        return switch (type) {
+            case CASSANDRA_TABLE -> CassandraTableConfig.class;
+            case CASSANDRA_KEYSPACE -> CassandraKeyspaceConfig.class;
+            case ASTRA_KEYSPACE -> AstraKeyspaceConfig.class;
+            default -> throw new IllegalArgumentException("Unknown asset type " + type);
+        };
     }
 
     @Override
     protected void validateAsset(AssetDefinition assetDefinition, Map<String, Object> asset) {
         Map<String, Object> configuration = ConfigurationUtils.getMap("config", null, asset);
-        requiredField(configuration, "datasource", describe(assetDefinition));
         final Map<String, Object> datasource =
                 ConfigurationUtils.getMap("datasource", Map.of(), configuration);
         final Map<String, Object> datasourceConfiguration =
                 ConfigurationUtils.getMap("configuration", Map.of(), datasource);
         switch (assetDefinition.getAssetType()) {
-            case "cassandra-table" -> {
-                requiredNonEmptyField(configuration, "table-name", describe(assetDefinition));
-                requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition));
-                requiredListField(configuration, "create-statements", describe(assetDefinition));
+            case CASSANDRA_TABLE -> {
                 checkDeleteStatements(assetDefinition, configuration);
             }
-            case "cassandra-keyspace" -> {
-                requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition));
-                requiredListField(configuration, "create-statements", describe(assetDefinition));
+            case CASSANDRA_KEYSPACE -> {
                 checkDeleteStatements(assetDefinition, configuration);
                 if (datasourceConfiguration.containsKey("secureBundle")
                         || datasourceConfiguration.containsKey("database")) {
                     throw new IllegalArgumentException("Use astra-keyspace for AstraDB services");
                 }
             }
-            case "astra-keyspace" -> {
-                requiredNonEmptyField(configuration, "keyspace", describe(assetDefinition));
+            case ASTRA_KEYSPACE -> {
                 if (!datasourceConfiguration.containsKey("secureBundle")
                         && !datasourceConfiguration.containsKey("database")) {
                     throw new IllegalArgumentException(
@@ -70,8 +81,7 @@ protected void validateAsset(AssetDefinition assetDefinition, Map<String, Object
                 requiredNonEmptyField(
                         datasourceConfiguration, "database", describe(assetDefinition));
             }
-            default -> throw new IllegalStateException(
-                    "Unexpected value: " + assetDefinition.getAssetType());
+            default -> {}
         }
     }
 
@@ -89,6 +99,127 @@ private static void checkDeleteStatements(
 
     @Override
     protected boolean lookupResource(String fieldName) {
-        return "datasource".contains(fieldName);
+        return "datasource".equals(fieldName);
+    }
+
+    @AssetConfig(
+            name = "Cassandra table",
+            description =
+                    """
+                    Manage a Cassandra table in existing keyspace.
+                    """)
+    @Data
+    public static class CassandraTableConfig {
+
+        @ConfigProperty(
+                description =
+                        """
+                       Reference to a datasource id configured in the application.
+                       """,
+                required = true)
+        private String datasource;
+
+        @ConfigProperty(
+                description =
+                        """
+                       Name of the table.
+                       """,
+                required = true)
+        @JsonProperty("table-name")
+        private String table;
+
+        @ConfigProperty(
+                description =
+                        """
+                       Name of the keyspace where the table is located.
+                       """,
+                required = true)
+        private String keyspace;
+
+        @ConfigProperty(
+                description =
+                        """
+                       List of the statement to execute to create the table. They will be executed every time the application is deployed or upgraded.
+                       """,
+                required = true)
+        @JsonProperty("create-statements")
+        private List<String> createStatements;
+
+        @ConfigProperty(
+                description =
+                        """
+                       List of the statement to execute to cleanup the table. They will be executed when the application is deleted only if 'deletion-mode' is 'delete'.
+                       """)
+        @JsonProperty("delete-statements")
+        private List<String> deleteStatements;
+    }
+
+    @AssetConfig(
+            name = "Cassandra keyspace",
+            description =
+                    """
+                    Manage a Cassandra keyspace.
+                    """)
+    @Data
+    public static class CassandraKeyspaceConfig {
+
+        @ConfigProperty(
+                description =
+                        """
+                       Reference to a datasource id configured in the application.
+                       """,
+                required = true)
+        private String datasource;
+
+        @ConfigProperty(
+                description =
+                        """
+                       Name of the keyspace to create.
+                       """,
+                required = true)
+        private String keyspace;
+
+        @ConfigProperty(
+                description =
+                        """
+                       List of the statement to execute to create the keyspace. They will be executed every time the application is deployed or upgraded.
+                       """,
+                required = true)
+        @JsonProperty("create-statements")
+        private List<String> createStatements;
+
+        @ConfigProperty(
+                description =
+                        """
+                       List of the statement to execute to cleanup the keyspace. They will be executed when the application is deleted only if 'deletion-mode' is 'delete'.
+                       """)
+        @JsonProperty("delete-statements")
+        private List<String> deleteStatements;
+    }
+
+    @AssetConfig(
+            name = "Astra keyspace",
+            description =
+                    """
+                    Manage a DataStax Astra keyspace.
+                    """)
+    @Data
+    public static class AstraKeyspaceConfig {
+
+        @ConfigProperty(
+                description =
+                        """
+                       Reference to a datasource id configured in the application.
+                       """,
+                required = true)
+        private String datasource;
+
+        @ConfigProperty(
+                description =
+                        """
+                       Name of the keyspace to create.
+                       """,
+                required = true)
+        private String keyspace;
     }
 }