Add ability to inject define macros into preprocessor. #174

systemrdl/compiler.py

@@ -179,7 +179,7 @@ def list_udps(self) -> List[str]:
         return udps
 
 
-    def preprocess_file(self, path: str, incl_search_paths: Optional[List[str]]=None) -> FileInfo:
+    def preprocess_file(self, path: str, incl_search_paths: Optional[List[str]]=None, defines: Optional[Dict[str,str]]=None) -> FileInfo:
         """
         Preprocess a single file without compiling it.
 
@@ -197,6 +197,10 @@ def preprocess_file(self, path: str, incl_search_paths: Optional[List[str]]=None
             1. Search each path specified in ``incl_search_paths``.
             2. Path relative to the source file performing the include.
 
+        defines: Dict[str, str]
+            Dictionary of pre-defined verilog macros where the key is the macro
+            name, and the value is the macro text.
+
         Raises
         ------
         RDLCompileError
@@ -213,12 +217,12 @@ def preprocess_file(self, path: str, incl_search_paths: Optional[List[str]]=None
         if incl_search_paths is None:
             incl_search_paths = []
 
-        input_stream, included_files = preprocessor.preprocess_file(self.env, path, incl_search_paths)
+        input_stream, included_files = preprocessor.preprocess_file(self.env, path, incl_search_paths, defines)
 
         return FileInfo(input_stream.strdata, included_files)
 
 
-    def compile_file(self, path: str, incl_search_paths: Optional[List[str]]=None) -> FileInfo:
+    def compile_file(self, path: str, incl_search_paths: Optional[List[str]]=None, defines: Optional[Dict[str,str]]=None) -> FileInfo:
         """
         Parse & compile a single file and append it to RDLCompiler's root
         namespace.
@@ -240,6 +244,10 @@ def compile_file(self, path: str, incl_search_paths: Optional[List[str]]=None) -
             1. Search each path specified in ``incl_search_paths``.
             2. Path relative to the source file performing the include.
 
+        defines: Dict[str, str]
+            Dictionary of pre-defined verilog macros where the key is the macro
+            name, and the value is the macro text.
+
         Raises
         ------
         RDLCompileError
@@ -258,7 +266,7 @@ def compile_file(self, path: str, incl_search_paths: Optional[List[str]]=None) -
         if incl_search_paths is None:
             incl_search_paths = []
 
-        input_stream, included_files = preprocessor.preprocess_file(self.env, path, incl_search_paths)
+        input_stream, included_files = preprocessor.preprocess_file(self.env, path, incl_search_paths, defines)
 
         # Run Antlr parser on input
         parsed_tree = sa_systemrdl.parse(

systemrdl/preprocessor/__init__.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, List, Set, Tuple
+from typing import TYPE_CHECKING, List, Set, Tuple, Optional, Dict
 
 from .perl_preprocessor import PerlPreprocessor
 from .verilog_preprocessor import VerilogPreprocessor
@@ -8,14 +8,14 @@
 if TYPE_CHECKING:
     from ..compiler import RDLEnvironment
 
-def preprocess_file(env: 'RDLEnvironment', path: str, search_paths: List[str]) -> Tuple[PreprocessedInputStream, Set[str]]:
+def preprocess_file(env: 'RDLEnvironment', path: str, search_paths: List[str], defines: Optional[Dict[str,str]]=None) -> Tuple[PreprocessedInputStream, Set[str]]:
     # Run file through Perl preprocessor
     ppp = PerlPreprocessor(env, path, search_paths)
     preprocessed_text, seg_map = ppp.preprocess()
     included_files = ppp.included_files
 
     # ... then through the Verilog preprocessor
-    vpp = VerilogPreprocessor(env, preprocessed_text, seg_map)
+    vpp = VerilogPreprocessor(env, preprocessed_text, seg_map, defines=defines)
     preprocessed_text, seg_map = vpp.preprocess()
 
     #segment_map.print_segment_debug(preprocessed_text, seg_map)

systemrdl/preprocessor/verilog_preprocessor.py

@@ -1,15 +1,14 @@
 import re
-from typing import List, Union, Tuple, TYPE_CHECKING, Optional, Match
+from typing import List, Union, Tuple, TYPE_CHECKING, Optional, Match, Dict
 
 from ..source_ref import SourceRefBase, SegmentedSourceRef
 from . import segment_map
 
 if TYPE_CHECKING:
     from ..compiler import RDLEnvironment
-    from typing import Dict
 
 class VerilogPreprocessor:
-    def __init__(self, env: 'RDLEnvironment', text: str, src_seg_map: Optional[segment_map.SegmentMap]=None, src_ref_override: Optional[SourceRefBase]=None):
+    def __init__(self, env: 'RDLEnvironment', text: str, src_seg_map: Optional[segment_map.SegmentMap]=None, src_ref_override: Optional[SourceRefBase]=None, defines: Optional[Dict[str,str]]=None):
         self.env = env
 
         # Unprocessed text
@@ -36,6 +35,10 @@ def __init__(self, env: 'RDLEnvironment', text: str, src_seg_map: Optional[segme
 
         # Macro namespace
         self._macro_defs = {} # type: Dict[str, Macro]
+        if defines is not None:
+            for k, v in defines.items():
+                macro = Macro(v, [])
+                self._macro_defs[k] = macro
 
         self._conditional = ConditionalState()
         self._conditional_stack = [] # type: List[ConditionalState]

test/rdl_src/preprocessor.rdl

@@ -30,6 +30,7 @@ addrmap `TOP_NAME {
     `ifdef FOO
         reg2_t foo1;
         `INST(reg2_t, macro_foo1);
+        `FOO
     `elsif BAR
         reg2_t bar1;
         `INST(reg2_t, macro_bar1);
@@ -40,6 +41,11 @@ addrmap `TOP_NAME {
 
     `line 123 "asdf" 1
 
+    `ifdef BAR
+        reg2_t bar1p5;
+        `BAR
+    `endif
+
     `ifndef FOO
         reg2_t foo2;
     `elsif BAR

test/rdl_src/preprocessor_CRLF.rdl

@@ -30,6 +30,7 @@ addrmap `TOP_NAME {
     `ifdef FOO
         reg2_t foo1;
         `INST(reg2_t, macro_foo1);
+        `FOO
     `elsif BAR
         reg2_t bar1;
         `INST(reg2_t, macro_bar1);
@@ -40,6 +41,11 @@ addrmap `TOP_NAME {
 
     `line 123 "asdf" 1
 
+    `ifdef BAR
+        reg2_t bar1p5;
+        `BAR
+    `endif
+
     `ifndef FOO
         reg2_t foo2;
     `elsif BAR

test/test_preprocessor.py

@@ -10,7 +10,7 @@
 ])
 class TestPreprocessor(RDLSourceTestCase):
 
-    def test_preprocessor(self):
+    def test_perl_preprocessor(self):
         root = self.compile(
             [self.src],
             "top",
@@ -36,6 +36,111 @@ def test_preprocessor(self):
             self.assertEqual(reg1_data4.msb, 5)
             self.assertEqual(reg1_data4.lsb, 4)
 
+
+    def test_verilog_preprocessor(self):
+        root = self.compile(
+            [self.src],
+            "top",
+            incl_search_paths=["rdl_src/incdir"]
+        )
+
+        with self.subTest("conditional1"):
+            self.assertIsNone(root.find_by_path("top.foo1"))
+            self.assertIsNone(root.find_by_path("top.bar1"))
+            self.assertIsNotNone(root.find_by_path("top.else1"))
+            self.assertIsNone(root.find_by_path("top.macro_foo1"))
+            self.assertIsNone(root.find_by_path("top.macro_bar1"))
+            self.assertIsNotNone(root.find_by_path("top.macro_else1"))
+
+        with self.subTest("conditional2"):
+            self.assertIsNotNone(root.find_by_path("top.foo2"))
+            self.assertIsNone(root.find_by_path("top.bar2"))
+            self.assertIsNone(root.find_by_path("top.else2"))
+
+        with self.subTest("conditional3"):
+            self.assertIsNotNone(root.find_by_path("top.foo3"))
+            self.assertIsNone(root.find_by_path("top.bar3"))
+            self.assertIsNone(root.find_by_path("top.else3"))
+            self.assertIsNone(root.find_by_path("top.xxx3"))
+            self.assertIsNone(root.find_by_path("top.yyy3"))
+            self.assertIsNone(root.find_by_path("top.xyelse3"))
+
+        with self.subTest("conditional4"):
+            self.assertIsNone(root.find_by_path("top.foo4"))
+            self.assertIsNotNone(root.find_by_path("top.bar4"))
+            self.assertIsNone(root.find_by_path("top.else4"))
+            self.assertIsNone(root.find_by_path("top.xxx4"))
+            self.assertIsNotNone(root.find_by_path("top.yyy4"))
+            self.assertIsNone(root.find_by_path("top.xyelse4"))
+
+        with self.subTest("Misc maros"):
+            self.assertIsNotNone(root.find_by_path("top.abcd1234"))
+
+            desc = root.find_by_path("top").get_property('desc')
+            self.assertEqual(desc, "left side: \"right side\"")
+
+            name = root.find_by_path("top").get_property('name')
+            self.assertEqual(name, "b + 1 + 42 + a")
+
+        with self.subTest("conditional undef"):
+            self.assertIsNotNone(root.find_by_path("top.macrox_exists1"))
+            self.assertIsNone(root.find_by_path("top.macrox_dne1"))
+            self.assertIsNone(root.find_by_path("top.macrox_exists2"))
+            self.assertIsNotNone(root.find_by_path("top.macrox_dne2"))
+
+
+    def test_user_defines(self):
+        with self.subTest("foo"):
+            root = self.compile(
+                [self.src],
+                "top",
+                incl_search_paths=["rdl_src/incdir"],
+                defines={
+                    "FOO": "reg2_t foo_upp;",
+                }
+            )
+            self.assertIsNotNone(root.find_by_path("top.foo1"))
+            self.assertIsNotNone(root.find_by_path("top.foo_upp"))
+            self.assertIsNone(root.find_by_path("top.bar1"))
+            self.assertIsNone(root.find_by_path("top.bar1p5"))
+            self.assertIsNone(root.find_by_path("top.foo2"))
+            self.assertIsNone(root.find_by_path("top.bar2"))
+
+        with self.subTest("bar"):
+            root = self.compile(
+                [self.src],
+                "top",
+                incl_search_paths=["rdl_src/incdir"],
+                defines={
+                    "BAR": "reg2_t bar_upp;",
+                }
+            )
+            self.assertIsNone(root.find_by_path("top.foo1"))
+            self.assertIsNone(root.find_by_path("top.foo_upp"))
+            self.assertIsNotNone(root.find_by_path("top.bar1"))
+            self.assertIsNotNone(root.find_by_path("top.bar1p5"))
+            self.assertIsNotNone(root.find_by_path("top.bar_upp"))
+            self.assertIsNotNone(root.find_by_path("top.foo2"))
+            self.assertIsNone(root.find_by_path("top.bar2"))
+
+        with self.subTest("foo bar"):
+            root = self.compile(
+                [self.src],
+                "top",
+                incl_search_paths=["rdl_src/incdir"],
+                defines={
+                    "BAR": "reg2_t bar_upp;",
+                    "FOO": "reg2_t foo_upp;",
+                }
+            )
+            self.assertIsNotNone(root.find_by_path("top.foo1"))
+            self.assertIsNotNone(root.find_by_path("top.foo_upp"))
+            self.assertIsNone(root.find_by_path("top.bar1"))
+            self.assertIsNotNone(root.find_by_path("top.bar1p5"))
+            self.assertIsNotNone(root.find_by_path("top.bar_upp"))
+            self.assertIsNone(root.find_by_path("top.foo2"))
+            self.assertIsNotNone(root.find_by_path("top.bar2"))
+
     def test_src_ref_translation(self):
         root = self.compile(
             [self.src],
@@ -112,47 +217,3 @@ def test_src_ref_translation(self):
             self.assertEqual(os.path.basename(src_ref.path), "preprocessor_incl2.rdl")
             self.assertEqual(src_ref.line, 3)
             self.assertEqual(src_ref.line_selection, (12, 12))
-
-        with self.subTest("conditional1"):
-            self.assertIsNone(root.find_by_path("top.foo1"))
-            self.assertIsNone(root.find_by_path("top.bar1"))
-            self.assertIsNotNone(root.find_by_path("top.else1"))
-            self.assertIsNone(root.find_by_path("top.macro_foo1"))
-            self.assertIsNone(root.find_by_path("top.macro_bar1"))
-            self.assertIsNotNone(root.find_by_path("top.macro_else1"))
-
-        with self.subTest("conditional2"):
-            self.assertIsNotNone(root.find_by_path("top.foo2"))
-            self.assertIsNone(root.find_by_path("top.bar2"))
-            self.assertIsNone(root.find_by_path("top.else2"))
-
-        with self.subTest("conditional3"):
-            self.assertIsNotNone(root.find_by_path("top.foo3"))
-            self.assertIsNone(root.find_by_path("top.bar3"))
-            self.assertIsNone(root.find_by_path("top.else3"))
-            self.assertIsNone(root.find_by_path("top.xxx3"))
-            self.assertIsNone(root.find_by_path("top.yyy3"))
-            self.assertIsNone(root.find_by_path("top.xyelse3"))
-
-        with self.subTest("conditional4"):
-            self.assertIsNone(root.find_by_path("top.foo4"))
-            self.assertIsNotNone(root.find_by_path("top.bar4"))
-            self.assertIsNone(root.find_by_path("top.else4"))
-            self.assertIsNone(root.find_by_path("top.xxx4"))
-            self.assertIsNotNone(root.find_by_path("top.yyy4"))
-            self.assertIsNone(root.find_by_path("top.xyelse4"))
-
-        with self.subTest("Misc maros"):
-            self.assertIsNotNone(root.find_by_path("top.abcd1234"))
-
-            desc = root.find_by_path("top").get_property('desc')
-            self.assertEqual(desc, "left side: \"right side\"")
-
-            name = root.find_by_path("top").get_property('name')
-            self.assertEqual(name, "b + 1 + 42 + a")
-
-        with self.subTest("conditional undef"):
-            self.assertIsNotNone(root.find_by_path("top.macrox_exists1"))
-            self.assertIsNone(root.find_by_path("top.macrox_dne1"))
-            self.assertIsNone(root.find_by_path("top.macrox_exists2"))
-            self.assertIsNotNone(root.find_by_path("top.macrox_dne2"))

test/unittest_utils.py

@@ -30,15 +30,15 @@ def setUp(self):
         self.compiler_warning_flags = 0
         self.compiler_error_flags = 0
 
-    def compile(self, files, top_name=None, inst_name=None, parameters=None, incl_search_paths=None):
+    def compile(self, files, top_name=None, inst_name=None, parameters=None, incl_search_paths=None, defines=None):
         this_dir = os.path.dirname(os.path.realpath(__file__))
         rdlc = RDLCompiler(
             message_printer=TestPrinter(),
             warning_flags=self.compiler_warning_flags,
             error_flags=self.compiler_error_flags
         )
         for file in files:
-            rdlc.compile_file(os.path.join(this_dir, file), incl_search_paths)
+            rdlc.compile_file(os.path.join(this_dir, file), incl_search_paths, defines)
         return rdlc.elaborate(top_name, inst_name, parameters)