Merge pull request #32398 from vespa-engine/interns/magnus/documentat…

…ion-java

Interns/magnus/documentation java

.gitignore

@@ -55,4 +55,5 @@ install_manifest.txt
 *.cbp
 !/.copr/Makefile
 !/.buildkite/Makefile
-.vscode
+.vscode
+.metadata

integration/intellij/.gitignore

@@ -1,3 +1,4 @@
+bin/
 .gradle
 **/build/
 !src/**/build/

integration/schema-language-server/.gitignore

@@ -0,0 +1 @@
+bin/

integration/schema-language-server/clients/vscode/.vscodeignore

@@ -10,3 +10,4 @@ src/**
 out/
 node_modules
 esbuild.js
+out/

integration/schema-language-server/clients/vscode/package.json

@@ -65,7 +65,7 @@
     "test": "vscode-test",
     "check-types": "tsc --noEmit",
     "copy-images": "cp ../../resources/* ./images",
-    "langserver-install": "mkdir -p server && cp ../../language-server/target/schema-language-server-jar-with-dependencies.jar ./server/",
+    "langserver-install": "mkdir -p server && rm -r server/* && cp ../../language-server/target/schema-language-server-jar-with-dependencies.jar ./server/",
     "publish": "npm run compile && node out/publish.js"
   },
   "devDependencies": {

integration/schema-language-server/language-server/.gitignore

@@ -1,4 +1,5 @@
 effective-pom.xml
 debug.log
 .settings/
-__pycache__
+__pycache__
+tmp/

integration/schema-language-server/language-server/pom.xml

@@ -18,6 +18,16 @@
       <artifactId>org.eclipse.lsp4j</artifactId>
       <version>0.23.1</version>
     </dependency>
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+      <version>1.17.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.vladsch.flexmark</groupId>
+      <artifactId>flexmark-html2md-converter</artifactId>
+      <version>0.64.8</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>config-model</artifactId>
@@ -60,13 +70,6 @@
     </dependency>
   </dependencies>
   <build>
-
-    <resources>
-      <resource>
-        <directory>target/generated-resources</directory>
-      </resource>
-    </resources>
-
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -85,6 +88,50 @@
         <artifactId>maven-install-plugin</artifactId>
         <configuration><skip>true</skip></configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>fetch-documentation</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>java</goal>
+            </goals>
+            <configuration>
+                <mainClass>ai.vespa.schemals.documentation.FetchDocumentation</mainClass>
+                <arguments>
+                    <argument>${project.basedir}/target/generated-resources/hover</argument>
+                </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <!-- Important that this runs after FetchDocumentation -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>prepare-package</phase> 
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <!-- Copy from target/generated-resources to output-dir (resolves to target/classes)
+                   This is so the resources get detected by assemble-fat-jar -->
+              <outputDirectory>${project.build.outputDirectory}</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${project.basedir}/target/generated-resources</directory>
+                  <filtering>false</filtering>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>com.yahoo.vespa</groupId>
         <artifactId>bundle-plugin</artifactId>
@@ -102,25 +149,6 @@
         <groupId>com.github.os72</groupId>
         <artifactId>protoc-jar-maven-plugin</artifactId>
       </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-            <phase>generate-sources</phase>
-            <configuration>
-              <executable>${project.basedir}/src/main/python/buildDocs.sh</executable>
-              <arguments>
-                <argument>${project.basedir}/target/generated-resources/hover</argument>
-              </arguments>
-              <workingDirectory>src/main/python/</workingDirectory>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>   
         <groupId>org.congocc</groupId>   
         <artifactId>org.congocc.maven.plugin</artifactId>   

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java

@@ -1,7 +1,23 @@
 package ai.vespa.schemals;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Enumeration;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.stream.Collectors;
 
 import org.eclipse.lsp4j.CodeActionKind;
 import org.eclipse.lsp4j.CodeActionOptions;
@@ -24,6 +40,7 @@
 import org.eclipse.lsp4j.services.WorkspaceService;
 
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.documentation.FetchDocumentation;
 import ai.vespa.schemals.index.SchemaIndex;
 import ai.vespa.schemals.lsp.command.CommandRegistry;
 import ai.vespa.schemals.lsp.semantictokens.SchemaSemanticTokens;
@@ -35,6 +52,8 @@
  */
 public class SchemaLanguageServer implements LanguageServer, LanguageClientAware {
 
+    public static Path serverPath = null;
+
     private WorkspaceService workspaceService;
     private SchemaTextDocumentService textDocumentService;
     private SchemaDocumentScheduler schemaDocumentScheduler;
@@ -64,7 +83,7 @@ public SchemaLanguageServer() {
 
         this.textDocumentService = new SchemaTextDocumentService(this.logger, schemaDocumentScheduler, schemaIndex, schemaMessageHandler);
         this.workspaceService = new SchemaWorkspaceService(this.logger, schemaDocumentScheduler, schemaIndex, schemaMessageHandler);
-
+        serverPath = Paths.get(SchemaLanguageServer.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent();
     }    
 
     @Override
@@ -142,5 +161,95 @@ public void connect(LanguageClient languageClient) {
         this.schemaMessageHandler.connectClient(languageClient);
         this.client.logMessage(new MessageParams(MessageType.Log, "Language Server successfully connected to client."));
 
+        if (serverPath == null) return;
+
+        // Start a document fetching job in the background.
+        Path docPath = serverPath.resolve("hover");
+
+        try {
+            setupDocumentation(docPath);
+        } catch (IOException ioex) {
+            this.logger.error("Failed to set up documentation. Error: " + ioex.getMessage());
+        }
+    }
+
+    /**
+     * Initial setup of the documentation.
+     */
+    public void setupDocumentation(Path documentationPath) throws IOException {
+
+        Files.createDirectories(documentationPath);
+        Files.createDirectories(documentationPath.resolve("schema"));
+        Files.createDirectories(documentationPath.resolve("rankExpression"));
+
+        ensureLocalDocumentationLoaded(documentationPath);
+
+        Thread thread = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    FetchDocumentation.fetchDocs(documentationPath);
+                } catch(Exception e) {
+                    throw new RuntimeException(e.getMessage());
+                }
+            }
+        };
+        var logger = this.logger;
+        thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+            @Override
+            public void uncaughtException(Thread th, Throwable ex) {
+                logger.warning("Failed to fetch docs: " + ex.getMessage() + " is unavailable. Locally cached documentation will be used.");
+            }
+        });
+        logger.info("Fetching docs to path: " + documentationPath.toAbsolutePath().toString());
+        thread.start();
+
+    }
+
+    /**
+     * Assumes documentation is loaded if documentationPath/schema contains .md files.
+     * If documentation is not loaded, unpacks markdown files from the current jar.
+     */
+    private void ensureLocalDocumentationLoaded(Path documentationPath) throws IOException {
+        File dir = new File(documentationPath.resolve("schema").toString());
+        File[] contents = dir.listFiles(new FileFilter() {
+            @Override
+            public boolean accept(File pathname) {
+                return pathname.getName().endsWith(".md");
+            }
+        });
+        // Documentation exists
+        if (contents.length > 0) return;
+
+        logger.info("Extracting embedded documentation files.");
+
+        // If it doesn't exist, unpack from jar
+        var resources = Thread.currentThread().getContextClassLoader().getResources(documentationPath.getFileName().toString());
+
+        if (!resources.hasMoreElements()) {
+            throw new IOException("Could not find documentation in jar file!");
+        }
+
+        URL resourceURL = resources.nextElement();
+
+        if (!resourceURL.getProtocol().equals("jar")) {
+            throw new IOException("Unhandled protocol for resource " +  resourceURL.toString());
+        }
+
+        String jarPath = resourceURL.getPath().substring(5, resourceURL.getPath().indexOf('!'));
+        try (JarFile jarFile = new JarFile(URLDecoder.decode(jarPath, "UTF-8"))) {
+            Enumeration<JarEntry> entries = jarFile.entries();
+            while (entries.hasMoreElements()) {
+                JarEntry entry = entries.nextElement();
+                if (!entry.isDirectory() && entry.getName().startsWith(documentationPath.getFileName().toString())) {
+                    Path destination = documentationPath.getParent().resolve(entry.getName());
+                    try (InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(entry.getName())) {
+                        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+                        String content = reader.lines().collect(Collectors.joining(System.lineSeparator()));
+                        Files.write(destination, content.getBytes(), StandardOpenOption.CREATE);
+                    }
+                }
+            }
+        }
     }
 }