IP copy rtl files (#22)

* Save commit.

* sim sources are only fetched by cocotb func if TECH_FLOW is not set.

* test to solve ip_link fetching problem.

* Added include files copy and ip name extraction.

* inc dirs bug corrected.

* CI docs pages deploy test fix.

* Revert "test to solve ip_link fetching problem."

This reverts commit eea0eff.

* tmrg func code cleaned.

---------

Co-authored-by: Benoit Denkinger <[email protected]>

.github/workflows/pages.yml

@@ -31,11 +31,14 @@ jobs:
       - name: make the sphinx docs
         run: |
           # sphinx-apidoc -f -o docs/source . -H Test -e -t docs/source/_templates
-          make -C docs html
+          mkdir -p docs/build
+          cd docs/build
+          cmake ../
+          make api_doc
 
       - name: Init new repo in dist folder and commit generated files
         run: |
-          cd docs/_build/html/
+          cd html/
           git init
           touch .nojekyll
           git add -A

cmake/sim/cocotb/cocotb_iverilog.cmake

@@ -87,7 +87,10 @@ function(cocotb_iverilog IP_LIB)
 
     # Get the IP RTL sources
     get_ip_rtl_sources(SOURCES ${IP_LIB})
-    get_ip_sim_only_sources(SIM_SOURCES ${IP_LIB})
+    # Get the sim_only sources only if we are not running a technology-independent flow
+    if(NOT TECH_FLOW)
+        get_ip_sim_only_sources(SIM_SOURCES ${IP_LIB})
+    endif()
     list(PREPEND SOURCES ${SIM_SOURCES})
 
     # Get IP include directories
@@ -193,7 +196,7 @@ function(cocotb_iverilog IP_LIB)
         TOPLEVEL=${TOP_MODULE}
         TOPLEVEL_LANG=${TOPLEVEL_LANG}
     )
-    set(COCOTB_IVERILOG_CMD 
+    set(COCOTB_IVERILOG_CMD
         # iverilog run-time engine must be in your path
         vvp -M${COCOTB_LIB_DIR} -m${COCOTB_LIB_VPI_ICARUS}
         # Arguments to pass to execution of compiled simulation
@@ -213,7 +216,7 @@ function(cocotb_iverilog IP_LIB)
         DEPENDS ${COCOTB_TEST}
         COMMENT "Running cocotb simulation on ${IP_LIB}"
         )
-        
+
     # Set the command as a property to be easily found by add_test()
     string(TOUPPER ${COCOTB_TEST} COCOTB_TEST_PROP)
     set_target_properties(${IP_LIB} PROPERTIES COCOTB_IVERILOG_${COCOTB_TEST_PROP}_CMD "${COCOTB_IVERILOG_CMD}")

cmake/utils/copy_rtl_files/copy_rtl_files.cmake

@@ -29,9 +29,22 @@ function(copy_rtl_files IP_LIB)
         list(APPEND RTL_FILES_ARGS ${file})
     endforeach()
 
+    # Get the list of RTL include directories
+    get_ip_include_directories(INCLUDES ${IP_LIB} SYSTEMVERILOG)
+    # Create a list to hold the RTL directoires as arguments for the Python script
+    set(RTL_INC_DIRS_ARGS)
+    # Add each RTL directory to the list of arguments
+    foreach(dir ${INCLUDES})
+        list(APPEND RTL_INC_DIRS_ARGS ${dir})
+    endforeach()
+
     find_python3()
     set(__CMD ${Python3_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/copy_rtl_files.py
-        ${TOP_MODULE} --stats --outdir ${OUTDIR} ${RTL_FILES_ARGS}
+        ${TOP_MODULE}
+        --stats
+        --outdir ${OUTDIR}
+        --files ${RTL_FILES_ARGS}
+        --inc_dirs ${RTL_INC_DIRS_ARGS}
     )
 
     # Call the Python script with the output directory and the RTL files

cmake/utils/copy_rtl_files/copy_rtl_files.py

@@ -1,37 +1,50 @@
 import os
+import sys
 import shutil
 import argparse
+import re
 from tmrg.verilog_elaborator import VerilogElaborator
 
-def remove_common_path(file_paths):
-    # Find the common path
-    common_path = os.path.commonpath(file_paths)
-
-    # Get the last common directory
-    last_common_directory = os.path.basename(common_path)
+def find_src_dep_name(file_paths, inc_f = False):
+    # Regular expression pattern to match the desired substring
+    pattern = r'/([^/]+)-(build|src|subbuild)'
 
     # Iterate over each file path and remove the common path
-    updated_paths = []
+    dep_names = []
     for path in file_paths:
-        relative_path = os.path.relpath(path, common_path)
-        updated_path = os.path.join(last_common_directory, relative_path)
-        updated_paths.append(updated_path)
+        # Search for the pattern in the string
+        match = re.search(pattern, path)
+        if not match:
+            print(f'ERROR : dependency base name for file f{path} not found.')
+            sys.exit(1)
+        else:
+            dep_name = match.group(1)
+
+        # If we have a list of include directories we put the files to includes
+        if inc_f:
+            # Add the file to the dependencies list
+            dep_names.append(f'{dep_name}/includes')
+        else:
+            last_directory = os.path.basename(os.path.dirname(path))
+            # Add the file to the dependencies list
+            dep_names.append(f'{dep_name}/{last_directory}/{os.path.basename(path)}')
 
-    return updated_paths
+    return dep_names
 
-def parse_input_files(velab, input_files):
+def parse_input_files(velab):
 
     def _enterH(velab, module, i="", done=[], hierarchy_files=[], hierarchy_missing_files=[]):
         i += "  |"
         for instName, inst in velab.global_namespace.modules[module].instances:
             if inst.module_name in velab.global_namespace.modules:
                 print(i+"- "+instName+":"+inst.module_name)
-                hierarchy_files.append(velab.global_namespace.modules[inst.module_name].file.filename)
-                if id(inst) in done:
-                    continue
-                else:
+                if id(inst) not in done:
                     done.append(id(inst))
                     _enterH(velab, inst.module_name, i, done, hierarchy_files, hierarchy_missing_files)
+                # Add the file after so the lowest level modules are first in the list
+                module_file_path = velab.global_namespace.modules[inst.module_name].file.filename
+                if module_file_path not in hierarchy_files:
+                    hierarchy_files.append(module_file_path)
             else:
                 print(i+"- [!] "+instName+":"+inst.module_name)
                 if inst.module_name not in hierarchy_missing_files:
@@ -42,18 +55,18 @@ def _enterH(velab, module, i="", done=[], hierarchy_files=[], hierarchy_missing_
     hierarchy_missing_files = []
     _enterH(velab, velab.topModule, "", done, hierarchy_files, hierarchy_missing_files)
 
-    # for module in hierarchy_files:
-    #     print(f"Module {module} found")
+    # Add the top file to the list
+    hierarchy_files.append(velab.global_namespace.modules[velab.topModule].file.filename)
 
     for module in hierarchy_missing_files:
         print(f"WARNING: Module {module} missing")
 
     return hierarchy_files
 
-
 def main():
+
     parser = argparse.ArgumentParser(description="Filter RTL files based on module hierarchy.")
-    # parser.add_argument('--top', help='Top module name')
+
     parser.add_argument('--outdir', help='Output directory where files will be copied')
     parser.add_argument('--list-files', action="store_true", help="Don't generate files, but instead just list the files that will be generated")
     parser.add_argument("--inc-dir", dest="inc_dir", action="append", default=[],
@@ -64,7 +77,8 @@ def main():
                         action="store_true", default=False, help="Generate bug report")
     parser.add_argument("--stats", dest="stats", action="store_true", help="Print statistics")
     parser.add_argument("--top-module", dest="top_module", action="store", default="", help="Specify top module name")
-    parser.add_argument('files', metavar='F', type=str, nargs='+', help='List of RTL file paths')
+    parser.add_argument('--files', metavar='F', type=str, nargs='+', help='List of RTL file paths')
+    parser.add_argument('--inc_dirs', metavar='F', type=str, nargs='+', help='List of RTL include directories')
 
     args = parser.parse_args()
 
@@ -81,37 +95,85 @@ def main():
         velab.elaborate(allowMissingModules=True)
         # Get on the files from the hierarchy if top module is provided
         if args.top_module:
-            files = parse_input_files(velab, files)
-
+            files = parse_input_files(velab)
 
         # Remove common path of the files
-        files_striped = remove_common_path(files)
+        src_names_list = find_src_dep_name(files)
+
+        # Get the include directories
+        inc_dirs = args.inc_dirs
+        # Remove common path of the inc_dirs
+        inc_names_list = find_src_dep_name(inc_dirs, True)
+
         # Get absolute path of the dst directory
         abs_path = os.path.abspath(args.outdir)
-        # Change file paths to dst directory
+
+        # Add absolute path to file destinations
         files_dst = []
-        for f_dst in files_striped:
+        for f_dst in src_names_list:
             files_dst.append(abs_path + '/' + f_dst)
+        inc_dirs_dst = []
+        for inc_dst in inc_names_list:
+            inc_dirs_dst.append(abs_path + '/' + inc_dst)
 
         # Only print the files if argument is passed
         if args.list_files:
-            print(*files_dst)
+            for f_src, f_dst in zip(files, files_dst):
+                print(f"{f_src} \n-> {f_dst}")
+            for inc_src, inc_dst in zip(inc_dirs, inc_dirs_dst):
+                print(f"{inc_src} \n-> {inc_dst}")
         else:
-            file_list_path = abs_path + '/file_list.txt'
+            file_list_path = abs_path + '/deps_file_list.py'
             # Check if the file list exist, otherwise create it
             os.makedirs(os.path.dirname(file_list_path), exist_ok=True)
             # Write the files to the file list
             with open(file_list_path, 'w') as outfile:
+                # Write the rtl files
+                outfile.write('rtl_deps_files = [\n')
                 for f_src, f_dst in zip(files, files_dst):
                     # Write the path to the list of path file
-                    outfile.write(f_dst + '\n')
+                    outfile.write(f'  "{f_dst}",\n')
                     # Create the folder hierarchy
                     os.makedirs(os.path.dirname(f_dst), exist_ok=True)
                     # Copy the file to the new location
                     shutil.copy2(f_src, f_dst)
+                # Close the rtl_deps_files list
+                outfile.write(']\n')
+                # Add empty line at the end of the file
+                outfile.write('\n')
 
+                # Write the include directories
+                any_inc_file_found = False
+                for inc_src, inc_dst in zip(inc_dirs, inc_dirs_dst):
+                    current_inc_file_found = False
+                    # Copy the file to src directory)
+                    for inc_file in os.listdir(inc_src):
+                        # Only copy files not directories
+                        inc_src_abs_path = os.path.join(inc_src, inc_file)
+                        inc_dst_abs_path = os.path.join(inc_dst, inc_file)
+                        if os.path.isfile(inc_src_abs_path):
+                            _, file_ext = os.path.splitext(inc_file)
+                            # We only copy files ending with svh
+                            if file_ext == '.svh':
+                                # Add the list when first inc file is found
+                                if not any_inc_file_found:
+                                    outfile.write('rtl_deps_incdirs = [\n')
+                                    any_inc_file_found = True
+                                # Add the path when the first valid inc file is found for the current include directory
+                                if not current_inc_file_found:
+                                    # Create the folder hierarchy
+                                    os.makedirs(os.path.dirname(inc_dst), exist_ok=True)
+                                    # Add the include path to the output file
+                                    outfile.write(f'  "{inc_src}",\n')
+                                    current_inc_file_found = True
+                                # Copy the file to the new location
+                                shutil.copy2(inc_src_abs_path, inc_dst_abs_path)
+                # Close the rtl_deps_incdirs list
+                if any_inc_file_found:
+                    outfile.write(']\n')
                 # Add empty line at the end of the file
                 outfile.write('\n')
+
             print(f"Filtered file list written to {file_list_path}")
     except ValueError as e:
         print(e)