Merge pull request #76 from FalkorDB/staging

Staging

Author: AviAvni

api/analyzers/analyzer.py

@@ -16,12 +16,54 @@ def find_parent(self, node: Node, parent_types: list) -> Node:
         while node and node.type not in parent_types:
             node = node.parent
         return node
+    
+    @abstractmethod
+    def is_dependency(self, file_path: str) -> bool:
+        """
+        Check if the file is a dependency.
+
+        Args:
+            file_path (str): The file path.
+
+        Returns:
+            bool: True if the file is a dependency, False otherwise.
+        """
+
+        pass
+    
+    @abstractmethod
+    def resolve_path(self, file_path: str, path: Path) -> str:
+        """
+        Resolve the path of the file.
+
+        Args:
+            file_path (str): The file path.
+            path (Path): The path to the folder.
+
+        Returns:
+            str: The resolved path.
+        """
 
-    def resolve(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[tuple[File, Node]]:
+        pass
+
+    def resolve(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, node: Node) -> list[tuple[File, Node]]:
         try:
-            return [(files[Path(location['absolutePath'])], files[Path(location['absolutePath'])].tree.root_node.descendant_for_point_range(Point(location['range']['start']['line'], location['range']['start']['character']), Point(location['range']['end']['line'], location['range']['end']['character']))) for location in lsp.request_definition(str(path), node.start_point.row, node.start_point.column) if location and Path(location['absolutePath']) in files]
+            locations = lsp.request_definition(str(file_path), node.start_point.row, node.start_point.column)
+            return [(files[Path(self.resolve_path(location['absolutePath'], path))], files[Path(self.resolve_path(location['absolutePath'], path))].tree.root_node.descendant_for_point_range(Point(location['range']['start']['line'], location['range']['start']['character']), Point(location['range']['end']['line'], location['range']['end']['character']))) for location in locations if location and Path(self.resolve_path(location['absolutePath'], path)) in files]
         except Exception as e:
             return []
+        
+    @abstractmethod
+    def add_dependencies(self, path: Path, files: dict[Path, File]):
+        """
+        Add dependencies to the files.
+
+        Args:
+            path (Path): The path to the folder.
+            files (dict[Path, File]): The files.
+        """
+
+        pass
 
     @abstractmethod
     def get_entity_label(self, node: Node) -> str:
@@ -85,7 +127,7 @@ def add_symbols(self, entity: Entity) -> None:
         pass
 
     @abstractmethod
-    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, key: str, symbol: Node) -> Entity:
+    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, key: str, symbol: Node) -> Entity:
         """
         Resolve a symbol to an entity.
 

api/analyzers/c/analyzer.py

@@ -1,7 +1,7 @@
-# import io
 # import os
-# from ..utils import *
 # from pathlib import Path
+
+# from multilspy import SyncLanguageServer
 # from ...entities import *
 # from ...graph import Graph
 # from typing import Optional
@@ -17,6 +17,30 @@
 #     def __init__(self) -> None:
 #         super().__init__(Language(tsc.language()))
 
+#     def get_entity_label(self, node: Node) -> str:
+#         if node.type == 'struct_specifier':
+#             return "Struct"
+#         elif node.type == 'function_definition':
+#             return "Function"
+#         raise ValueError(f"Unknown entity type: {node.type}")
+
+#     def get_entity_name(self, node: Node) -> str:
+#         if node.type in ['struct_specifier', 'function_definition']:
+#             return node.child_by_field_name('name').text.decode('utf-8')
+#         raise ValueError(f"Unknown entity type: {node.type}")
+    
+#     def get_entity_docstring(self, node: Node) -> Optional[str]:
+#         if node.type in ['struct_specifier', 'function_definition']:
+#             body = node.child_by_field_name('body')
+#             if body.child_count > 0 and body.children[0].type == 'expression_statement':
+#                 docstring_node = body.children[0].child(0)
+#                 return docstring_node.text.decode('utf-8')
+#             return None
+#         raise ValueError(f"Unknown entity type: {node.type}")  
+
+#     def get_entity_types(self) -> list[str]:
+#         return ['struct_specifier', 'function_definition']
+
 #     def process_pointer_declaration(self, node: Node) -> tuple[str, int]:
 #         """
 #         Processes a pointer declaration node to determine the argument name and pointer count.
@@ -313,78 +337,54 @@
 #             # Connect parent to entity
 #             graph.connect_entities('DEFINES', parent.id, entity.id)
 
-#     def first_pass(self, path: Path, graph:Graph) -> None:
-#         """
-#         Perform the first pass processing of a C source file or header file.
-
-#         Args:
-#             path (Path): The path to the C source file or header file.
-#             f (io.TextIOWrapper): The file object representing the opened C source file or header file.
-#             graph (Graph): The Graph object where entities will be added.
-
-#         Returns:
-#             None
-
-#         Raises:
-#             None
-
-#         This function processes the specified C source file or header file to extract and add function definitions
-#         and struct definitions to the provided graph object.
-
-#         - If the file path does not end with '.c' or '.h', it logs a debug message and skips processing.
-#         - It creates a File entity representing the file and adds it to the graph.
-#         - It parses the file content using a parser instance (`self.parser`).
-#         - Function definitions and struct definitions are extracted using Tree-sitter queries.
-#         - Each function definition is processed using `self.process_function_definition`.
-#         - Each struct definition is processed using `self.process_struct_specifier`.
-#         """
-
-#         if path.suffix != '.c' and path.suffix != '.h':
-#             logger.debug(f"Skipping none C file {path}")
-#             return
-
-#         logger.info(f"Processing {path}")
-
-#         # Create file entity
-#         file = File(os.path.dirname(path), path.name, path.suffix)
-#         graph.add_file(file)
-
-#         # Parse file
-#         source_code = path.read_bytes()
-#         tree = self.parser.parse(source_code)
-#         try:
-#             source_code = source_code.decode('utf-8')
-#         except Exception as e:
-#             logger.error(f"Failed decoding source code: {e}")
-#             source_code = ''
-
-#         # Process function definitions
-#         query = self.language.query("(function_definition) @function")
-#         captures = query.captures(tree.root_node)
-#         # captures: {'function':
-#         #   [<Node type=function_definition, start_point=(0, 0), end_point=(7, 1)>,
-#         #    <Node type=function_definition, start_point=(15, 0), end_point=(18, 1)>
-#         #   ]
-#         # }
-
-#         if 'function' in captures:
-#             functions = captures['function']
-#             for node in functions:
-#                 self.process_function_definition(file, node, path, graph, source_code)
-
-#         # Process struct definitions
-#         query = self.language.query("(struct_specifier) @struct")
-#         captures = query.captures(tree.root_node)
-        
-#         if 'struct' in captures:
-#             structs = captures['struct']
-#             # captures: {'struct':
-#             #   [
-#             #       <Node type=struct_specifier, start_point=(9, 0), end_point=(13, 1)>
-#             #   ]
-#             # }
-#             for node in structs:
-#                 self.process_struct_specifier(file, node, path, graph)
+#     def add_symbols(self, entity: Entity) -> None:
+#         if entity.node.type == 'struct_specifier':
+#             superclasses = entity.node.child_by_field_name("superclasses")
+#             if superclasses:
+#                 base_classes_query = self.language.query("(argument_list (_) @base_class)")
+#                 base_classes_captures = base_classes_query.captures(superclasses)
+#                 if 'base_class' in base_classes_captures:
+#                     for base_class in base_classes_captures['base_class']:
+#                         entity.add_symbol("base_class", base_class)
+#         elif entity.node.type == 'function_definition':
+#             query = self.language.query("(call) @reference.call")
+#             captures = query.captures(entity.node)
+#             if 'reference.call' in captures:
+#                 for caller in captures['reference.call']:
+#                     entity.add_symbol("call", caller)
+#             query = self.language.query("(typed_parameter type: (_) @parameter)")
+#             captures = query.captures(entity.node)
+#             if 'parameter' in captures:
+#                 for parameter in captures['parameter']:
+#                     entity.add_symbol("parameters", parameter)
+#             return_type = entity.node.child_by_field_name('return_type')
+#             if return_type:
+#                 entity.add_symbol("return_type", return_type)
+
+#     def resolve_type(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+#         res = []
+#         for file, resolved_node in self.resolve(files, lsp, path, node):
+#             type_dec = self.find_parent(resolved_node, ['struct_specifier'])
+#             res.append(file.entities[type_dec])
+#         return res
+
+#     def resolve_method(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+#         res = []
+#         for file, resolved_node in self.resolve(files, lsp, path, node):
+#             method_dec = self.find_parent(resolved_node, ['function_definition'])
+#             if not method_dec:
+#                 continue
+#             if method_dec in file.entities:
+#                 res.append(file.entities[method_dec])
+#         return res
+    
+#     def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, key: str, symbol: Node) -> Entity:
+#         if key in ["parameters", "return_type"]:
+#             return self.resolve_type(files, lsp, path, symbol)
+#         elif key in ["call"]:
+#             return self.resolve_method(files, lsp, path, symbol)
+#         else:
+#             raise ValueError(f"Unknown key {key}")
 
 #     def second_pass(self, path: Path, graph: Graph) -> None:
 #         """

api/analyzers/java/analyzer.py

@@ -1,4 +1,6 @@
+import os
 from pathlib import Path
+import subprocess
 from ...entities import *
 from typing import Optional
 from ..analyzer import AbstractAnalyzer
@@ -8,13 +10,32 @@
 import tree_sitter_java as tsjava
 from tree_sitter import Language, Node
 
+from xml.etree import ElementTree
+
 import logging
 logger = logging.getLogger('code_graph')
 
 class JavaAnalyzer(AbstractAnalyzer):
     def __init__(self) -> None:
         super().__init__(Language(tsjava.language()))
 
+    def add_dependencies(self, path: Path, files: dict[Path, File]):
+        # if not Path("java-decompiler-engine-243.23654.153.jar").is_file():
+        #     subprocess.run(["wget", "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/java/java-decompiler-engine/243.23654.153/java-decompiler-engine-243.23654.153.jar"])
+        subprocess.run(["rm", "-rf", f"{path}/temp_deps"])
+        pom = ElementTree.parse(str(path) + '/pom.xml')
+        for dependency in pom.findall('.//{http://maven.apache.org/POM/4.0.0}dependency'):
+            groupId = dependency.find('{http://maven.apache.org/POM/4.0.0}groupId').text.replace('.', '/')
+            artifactId = dependency.find('{http://maven.apache.org/POM/4.0.0}artifactId').text
+            version = dependency.find('{http://maven.apache.org/POM/4.0.0}version').text
+            # jar_path = f"{Path.home()}/.m2/repository/{groupId}/{artifactId}/{version}/{artifactId}-{version}.jar"
+            jar_path = f"{Path.home()}/.m2/repository/{groupId}/{artifactId}/{version}/{artifactId}-{version}-sources.jar"
+
+            os.makedirs(f"{path}/temp_deps/{artifactId}-{version}", exist_ok=True)
+            # subprocess.run(["java", "-jar", "java-decompiler-engine-243.23654.153.jar", "-hdc=0 -iib=1 -rsy=1 -rbr=1 -dgs=1 -din=1 -den=1 -asc=1 -bsm=1", jar_path, f"{path}/temp_deps/{artifactId}-{version}"])
+            subprocess.run(["cp", jar_path, f"{artifactId}-{version}.jar"], cwd=f"{path}/temp_deps/{artifactId}-{version}")
+            subprocess.run(["unzip", f"{artifactId}-{version}.jar"], cwd=f"{path}/temp_deps/{artifactId}-{version}")
+
     def get_entity_label(self, node: Node) -> str:
         if node.type == 'class_declaration':
             return "Class"
@@ -75,28 +96,38 @@ def add_symbols(self, entity: Entity) -> None:
                         entity.add_symbol("parameters", parameter)
                 entity.add_symbol("return_type", entity.node.child_by_field_name('type'))
 
-    def resolve_type(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+    def is_dependency(self, file_path: str) -> bool:
+        return ".jar" in file_path
+
+    def resolve_path(self, file_path: str, path: Path) -> str:
+        if ".jar" in file_path:
+            args = file_path.replace(".jar", "").replace(".class", ".java").split("/")
+            targs = "/".join(["/".join(arg.split(".")) for arg in args[2:-1]])
+            return f"{path}/temp_deps/{args[1]}/{targs}/{args[-1]}"
+        return file_path
+
+    def resolve_type(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, node: Node) -> list[Entity]:
         res = []
-        for file, resolved_node in self.resolve(files, lsp, path, node):
+        for file, resolved_node in self.resolve(files, lsp, file_path, path, node):
             type_dec = self.find_parent(resolved_node, ['class_declaration', 'interface_declaration', 'enum_declaration'])
             if type_dec in file.entities:
                 res.append(file.entities[type_dec])
         return res
 
-    def resolve_method(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+    def resolve_method(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, node: Node) -> list[Entity]:
         res = []
-        for file, resolved_node in self.resolve(files, lsp, path, node.child_by_field_name('name')):
+        for file, resolved_node in self.resolve(files, lsp, file_path, path, node.child_by_field_name('name')):
             method_dec = self.find_parent(resolved_node, ['method_declaration', 'constructor_declaration', 'class_declaration', 'interface_declaration', 'enum_declaration'])
-            if method_dec.type in ['class_declaration', 'interface_declaration', 'enum_declaration']:
+            if method_dec and method_dec.type in ['class_declaration', 'interface_declaration', 'enum_declaration']:
                 continue
             if method_dec in file.entities:
                 res.append(file.entities[method_dec])
         return res
 
-    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, key: str, symbol: Node) -> Entity:
+    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, key: str, symbol: Node) -> Entity:
         if key in ["implement_interface", "base_class", "extend_interface", "parameters", "return_type"]:
-            return self.resolve_type(files, lsp, path, symbol)
+            return self.resolve_type(files, lsp, file_path, path, symbol)
         elif key in ["call"]:
-            return self.resolve_method(files, lsp, path, symbol)
+            return self.resolve_method(files, lsp, file_path, path, symbol)
         else:
             raise ValueError(f"Unknown key {key}")

api/analyzers/python/analyzer.py

@@ -1,3 +1,4 @@
+import subprocess
 from multilspy import SyncLanguageServer
 from pathlib import Path
 from ...entities import *
@@ -13,6 +14,16 @@
 class PythonAnalyzer(AbstractAnalyzer):
     def __init__(self) -> None:
         super().__init__(Language(tspython.language()))
+    
+    def add_dependencies(self, path: Path, files: dict[Path, File]):
+        if Path(f"{path}/venv").is_dir():
+            return
+        subprocess.run(["python3", "-m", "venv", f"{path}/venv"])
+        if Path(f"{path}/requirements.txt").is_file():
+            subprocess.run([f"{path}/venv/bin/pip", "install", "-r", "requirements.txt"])
+        if Path(f"{path}/pyproject.toml").is_file():
+            subprocess.run([f"{path}/venv/bin/pip", "install", "poetry"])
+            subprocess.run([f"{path}/venv/bin/poetry", "install"])
 
     def get_entity_label(self, node: Node) -> str:
         if node.type == 'class_definition':
@@ -62,27 +73,40 @@ def add_symbols(self, entity: Entity) -> None:
             if return_type:
                 entity.add_symbol("return_type", return_type)
 
-    def resolve_type(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+    def is_dependency(self, file_path: str) -> bool:
+        return "venv" in file_path
+
+    def resolve_path(self, file_path: str, path: Path) -> str:
+        return file_path
+
+    def resolve_type(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path, node: Node) -> list[Entity]:
         res = []
-        for file, resolved_node in self.resolve(files, lsp, path, node):
+        if node.type == 'attribute':
+            node = node.child_by_field_name('attribute')
+        for file, resolved_node in self.resolve(files, lsp, file_path, path, node):
             type_dec = self.find_parent(resolved_node, ['class_definition'])
-            res.append(file.entities[type_dec])
+            if type_dec in file.entities:
+                res.append(file.entities[type_dec])
         return res
 
-    def resolve_method(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, node: Node) -> list[Entity]:
+    def resolve_method(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, node: Node) -> list[Entity]:
         res = []
-        for file, resolved_node in self.resolve(files, lsp, path, node):
+        if node.type == 'call':
+            node = node.child_by_field_name('function')
+            if node.type == 'attribute':
+                node = node.child_by_field_name('attribute')
+        for file, resolved_node in self.resolve(files, lsp, file_path, path, node):
             method_dec = self.find_parent(resolved_node, ['function_definition', 'class_definition'])
             if not method_dec:
                 continue
             if method_dec in file.entities:
                 res.append(file.entities[method_dec])
         return res
 
-    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, path: Path, key: str, symbol: Node) -> Entity:
+    def resolve_symbol(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: Path, path: Path, key: str, symbol: Node) -> Entity:
         if key in ["base_class", "parameters", "return_type"]:
-            return self.resolve_type(files, lsp, path, symbol)
+            return self.resolve_type(files, lsp, file_path, path, symbol)
         elif key in ["call"]:
-            return self.resolve_method(files, lsp, path, symbol)
+            return self.resolve_method(files, lsp, file_path, path, symbol)
         else:
             raise ValueError(f"Unknown key {key}")