LSP: A bunch of new OpenGOAL language features (#3437)

- Integrate the AST into the LSP, this makes parsing and tokenizing the
files much easier
- Consolidate most of the symbol info tracking in `goalc` to a single
map. Fixed some issues where the old map would never evict symbols when
re-compiling files. There is still some more to cleanup, but this now
can be used as an incrementally updated source-of-truth for the LSP
- re-compile files when they are saved. Ideally this would be done
everytime they are changed but that:
  - may be too aggressive
- goalc doesn't compile incrementally yet so it likely would be a worse
UX

Features added, see
open-goal/opengoal-vscode#256
- Hover

![image](https://github.com/open-goal/jak-project/assets/13153231/58dadb5d-582c-4c1f-9ffe-eaa4c85a0255)

![image](https://github.com/open-goal/jak-project/assets/13153231/b383adde-57fc-462c-a256-b2de5c30ca9a)
- LSP Status fixed
- Type Hierarchy

![image](https://github.com/open-goal/jak-project/assets/13153231/8e681377-1d4e-4336-ad70-1695a4607340)
- Document Color

![image](https://github.com/open-goal/jak-project/assets/13153231/4e48ccd8-0ed1-4459-a133-5277561e4201)
- Document Symbols
![Screenshot 2024-03-27
004105](https://github.com/open-goal/jak-project/assets/13153231/8e655034-43c4-4261-b6e0-85de00cbfc7f)
- Completions
![Screenshot 2024-03-30
004504](https://github.com/open-goal/jak-project/assets/13153231/d123a187-af90-466b-9eb7-561b2ee97cd1)

---------

Co-authored-by: Hat Kid <[email protected]>

.github/workflows/windows-build-clang.yaml

@@ -61,7 +61,6 @@ jobs:
         run: cmake --build build --parallel %NUMBER_OF_PROCESSORS%
 
       - name: Run Tests
-        timeout-minutes: 10
         env:
           GTEST_OUTPUT: "xml:opengoal-test-report.xml"
         run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"

.vs/launch.vs.json

@@ -143,13 +143,6 @@
       "name": "Game - Jak 2 - Runtime (release)",
       "args": ["-v", "--game", "jak2", "--", "-boot", "-fakeiso"]
     },
-    {
-      "type": "default",
-      "project": "CMakeLists.txt",
-      "projectTarget": "goalc.exe (bin\\goalc.exe)",
-      "name": "REPL",
-      "args": ["--user-auto"]
-    },
     {
       "type": "default",
       "project": "CMakeLists.txt",

.vscode/settings.json

@@ -7,5 +7,6 @@
     },
     "editor.wordBasedSuggestions": "matchingDocuments",
     "editor.snippetSuggestions": "top"
-  }
+  },
+  "cmake.configureOnOpen": false
 }

common/CMakeLists.txt

@@ -66,6 +66,7 @@ add_library(common
         type_system/TypeSpec.cpp
         type_system/TypeSystem.cpp
         util/Assert.cpp
+        util/ast_util.cpp
         util/BitUtils.cpp
         util/compress.cpp
         util/crc32.cpp
@@ -87,7 +88,7 @@ add_library(common
         util/Timer.cpp
         util/unicode_util.cpp
         versions/versions.cpp
-        )
+         "util/trie_map.h")
 
 target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter sqlite3 libtinyfiledialogs)
 

common/formatter/formatter.cpp

@@ -2,8 +2,11 @@
 
 #include "formatter_tree.h"
 
+#include "common/formatter/rules/formatting_rules.h"
+#include "common/formatter/rules/rule_config.h"
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
+#include "common/util/ast_util.h"
 #include "common/util/string_util.h"
 
 #include "tree_sitter/api.h"
@@ -400,8 +403,6 @@ std::string join_formatted_lines(const std::vector<std::string>& lines,
 std::optional<std::string> formatter::format_code(const std::string& source) {
   // Create a parser.
   std::shared_ptr<TSParser> parser(ts_parser_new(), TreeSitterParserDeleter());
-
-  // Set the parser's language (JSON in this case).
   ts_parser_set_language(parser.get(), tree_sitter_opengoal());
 
   // Build a syntax tree based on source code stored in a string.

common/formatter/formatter.h

@@ -3,13 +3,8 @@
 #include <optional>
 #include <string>
 
-#include "common/formatter/rules/formatting_rules.h"
-#include "common/formatter/rules/rule_config.h"
-
 #include "tree_sitter/api.h"
 
-// TODO:
-// - Considering _eventually_ adding line-length heuristics
 namespace formatter {
 
 struct TreeSitterParserDeleter {

common/type_system/TypeSystem.cpp

@@ -1442,6 +1442,21 @@ std::vector<std::string> TypeSystem::search_types_by_parent_type(
   return results;
 }
 
+std::vector<std::string> TypeSystem::search_types_by_parent_type_strict(
+    const std::string& parent_type) {
+  std::vector<std::string> results = {};
+  for (const auto& [type_name, type_info] : m_types) {
+    // Only NullType's have no parent
+    if (!type_info->has_parent()) {
+      continue;
+    }
+    if (type_info->get_parent() == parent_type) {
+      results.push_back(type_name);
+    }
+  }
+  return results;
+}
+
 std::vector<std::string> TypeSystem::search_types_by_minimum_method_id(
     const int minimum_method_id,
     const std::optional<std::vector<std::string>>& existing_matches) {

common/type_system/TypeSystem.h

@@ -278,6 +278,7 @@ class TypeSystem {
   std::vector<std::string> search_types_by_parent_type(
       const std::string& parent_type,
       const std::optional<std::vector<std::string>>& existing_matches = {});
+  std::vector<std::string> search_types_by_parent_type_strict(const std::string& parent_type);
 
   std::vector<std::string> search_types_by_minimum_method_id(
       const int minimum_method_id,

common/util/FileUtil.cpp

@@ -736,4 +736,25 @@ std::string get_majority_file_line_endings(const std::string& file_contents) {
   return "\n";
 }
 
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents) {
+  size_t lf_count = 0;
+  size_t crlf_count = 0;
+
+  for (size_t i = 0; i < file_contents.size(); ++i) {
+    if (file_contents[i] == '\n') {
+      if (i > 0 && file_contents[i - 1] == '\r') {
+        crlf_count++;
+      } else {
+        lf_count++;
+      }
+    }
+  }
+
+  if (crlf_count > lf_count) {
+    return {lf_count + crlf_count, "\r\n"};
+  }
+  return {lf_count + crlf_count, "\n"};
+}
+
 }  // namespace file_util

common/util/FileUtil.h

@@ -72,4 +72,6 @@ std::vector<fs::path> sort_filepaths(const std::vector<fs::path>& paths, const b
 void copy_file(const fs::path& src, const fs::path& dst);
 std::string make_screenshot_filepath(const GameVersion game_version, const std::string& name = "");
 std::string get_majority_file_line_endings(const std::string& file_contents);
+std::pair<int, std::string> get_majority_file_line_endings_and_count(
+    const std::string& file_contents);
 }  // namespace file_util

common/util/Range.h

@@ -67,4 +67,4 @@ class Range {
  private:
   T m_start = {};
   T m_end = {};
-};
+};

common/util/Trie.h

@@ -12,7 +12,7 @@
  * It owns the memory for the objects it stores.
  * Doing an insert will create a copy of your object.
  *
- * Other that deleting the whole thing, there is no support for removing a node.
+ * Other than deleting the whole thing, there is no support for removing a node.
  */
 template <typename T>
 class Trie {

common/util/ast_util.cpp

@@ -0,0 +1,31 @@
+#include "ast_util.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node) {
+  uint32_t start = ts_node_start_byte(node);
+  uint32_t end = ts_node_end_byte(node);
+  return source.substr(start, end - start);
+}
+
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results) {
+  if (ts_node_child_count(curr_node) == 0) {
+    return;
+  }
+  std::vector<std::string> node_elements;
+  bool added = false;
+  for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
+    const auto child_node = ts_node_child(curr_node, i);
+    const auto contents = get_source_code(source, child_node);
+    node_elements.push_back(contents);
+    // Check for a match
+    if (node_elements == prefix && !added) {
+      results.push_back(curr_node);
+      added = true;
+    }
+    search_for_forms_that_begin_with(source, child_node, prefix, results);
+  }
+}
+}  // namespace ast_util

common/util/ast_util.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "tree_sitter/api.h"
+
+namespace ast_util {
+std::string get_source_code(const std::string& source, const TSNode& node);
+void search_for_forms_that_begin_with(const std::string& source,
+                                      const TSNode curr_node,
+                                      const std::vector<std::string>& prefix,
+                                      std::vector<TSNode>& results);
+
+}  // namespace ast_util

common/util/trie_map.h

@@ -0,0 +1,160 @@
+#pragma once
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// TrieMap class
+template <typename T>
+class TrieMap {
+ private:
+  // TrieNode structure
+  struct TrieNode {
+    std::unordered_map<char, std::shared_ptr<TrieNode>> children;
+    std::vector<std::shared_ptr<T>> elements;
+  };
+
+  std::shared_ptr<TrieNode> root;
+
+ public:
+  TrieMap() : root(std::make_shared<TrieNode>()) {}
+
+  // Insert an element with a key into the TrieMap and return the inserted element
+  std::shared_ptr<T> insert(const std::string& key, const T& element) {
+    std::shared_ptr<T> shared_element = std::make_shared<T>(element);
+    std::shared_ptr<TrieNode> node = root;
+    for (char c : key) {
+      if (node->children.find(c) == node->children.end()) {
+        node->children[c] = std::make_shared<TrieNode>();
+      }
+      node = node->children[c];
+    }
+    // Store element at the leaf node
+    node->elements.push_back(shared_element);
+    return shared_element;
+  }
+
+  // Retrieve elements with a given prefix
+  std::vector<std::shared_ptr<T>> retrieve_with_prefix(const std::string& prefix) const {
+    std::vector<std::shared_ptr<T>> result;
+    std::shared_ptr<TrieNode> node = root;
+    // Traverse to the node representing the prefix
+    for (char c : prefix) {
+      if (node->children.find(c) == node->children.end()) {
+        return result;  // No elements with the given prefix
+      }
+      node = node->children[c];
+    }
+    // Gather all elements stored at or below this node
+    retrieve_elements(node, result);
+    return result;
+  }
+
+  // Retrieve elements with an exact key match
+  std::vector<std::shared_ptr<T>> retrieve_with_exact(const std::string& key) const {
+    std::vector<std::shared_ptr<T>> result;
+    std::shared_ptr<TrieNode> node = root;
+    // Traverse to the node representing the key
+    for (char c : key) {
+      if (node->children.find(c) == node->children.end()) {
+        return result;  // No elements with the given key
+      }
+      node = node->children[c];
+    }
+    // Return elements stored at this node
+    return node->elements;
+  }
+
+  // Remove the specified element from the TrieMap
+  void remove(const std::shared_ptr<T>& element) { remove_element(root, element); }
+
+  // Return the total number of elements stored in the TrieMap
+  int size() const {
+    int count = 0;
+    count_elements(root, count);
+    return count;
+  }
+
+  // Return a vector containing shared pointers to all elements stored in the TrieMap
+  std::vector<std::shared_ptr<T>> get_all_elements() const {
+    std::vector<std::shared_ptr<T>> result;
+    get_all_elements_helper(root, result);
+    return result;
+  }
+
+ private:
+  // Recursive function to retrieve elements stored at or below the given node
+  void retrieve_elements(std::shared_ptr<TrieNode> node,
+                         std::vector<std::shared_ptr<T>>& result) const {
+    // Add elements stored at this node to the result
+    for (const auto& element : node->elements) {
+      result.push_back(element);
+    }
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      retrieve_elements(child.second, result);
+    }
+  }
+
+  // Recursive function to remove the specified element from the TrieMap
+  bool remove_element(std::shared_ptr<TrieNode> node, const std::shared_ptr<T>& element) {
+    // Remove the element if it exists at this node
+    auto& elements = node->elements;
+    auto it = std::find(elements.begin(), elements.end(), element);
+    if (it != elements.end()) {
+      elements.erase(it);
+      return true;
+    }
+    // Recursively search children
+    for (auto& child : node->children) {
+      if (remove_element(child.second, element)) {
+        // Remove child node if it's empty after removal
+        if (child.second->elements.empty() && child.second->children.empty()) {
+          node->children.erase(child.first);
+        }
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // Recursive function to count elements stored at or below the given node
+  void count_elements(std::shared_ptr<TrieNode> node, int& count) const {
+    // Increment count by the number of elements stored at this node
+    count += node->elements.size();
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      count_elements(child.second, count);
+    }
+  }
+
+  // Recursive helper function to collect all elements stored in the TrieMap
+  void get_all_elements_helper(std::shared_ptr<TrieNode> node,
+                               std::vector<std::shared_ptr<T>>& result) const {
+    // Add elements stored at this node to the result
+    for (const auto& element : node->elements) {
+      result.push_back(element);
+    }
+    // Recursively traverse children
+    for (const auto& child : node->children) {
+      get_all_elements_helper(child.second, result);
+    }
+  }
+};
+
+// TrieMap<std::string> trie_map;
+//
+//// Insert elements
+// std::shared_ptr<std::string> inserted_element_1 = trie_map.insert("apple", "A fruit");
+// std::shared_ptr<std::string> inserted_element_2 = trie_map.insert("app", "An application");
+// std::shared_ptr<std::string> inserted_element_3 = trie_map.insert("banana", "Another fruit");
+// std::shared_ptr<std::string> inserted_element_4 = trie_map.insert("apple", "Another apple");
+//
+//// Remove an element
+// trie_map.remove(inserted_element_1);
+//
+//// Retrieve elements with a prefix
+// std::vector<std::shared_ptr<std::string>> prefix_results = trie_map.retrieve_with_prefix("app");

goalc/CMakeLists.txt

@@ -32,6 +32,7 @@ add_library(compiler
         compiler/CompilerSettings.cpp
         compiler/CodeGenerator.cpp
         compiler/StaticObject.cpp
+        compiler/symbol_info.cpp
         compiler/compilation/Asm.cpp
         compiler/compilation/Atoms.cpp
         compiler/compilation/CompilerControl.cpp