Merge pull request #32 from Decompollaborate/develop

2.7.0

Author: AngheloAlf

CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.7.0] - 2024-09-24
+
+### Added
+
+- `MapFile.findSymbolByVram` and `MapFile.findSymbolByVrom` methods.
+  - Allow to search a symbol given a given address. The address will be treated
+    as either a vram address or a vrom address respectively.
+- Add `--vram`, `--vrom` and `--name` arguments to `sym_info` frontend.
+  - Allow to tell to `sym_info` exactly how to treat the argument instead of
+    trying to guess how to use it.
+- `sym_info` can now detect that an address may belong to a file even when the
+  symbol itself may not exist on the mapfile.
+  - This can happen for local symbols, for example for rodata literals.
+
+### Deprecated
+
+- `MapFile.findSymbolByVramOrVrom`.
+  - Use `MapFile.findSymbolByVram` and `MapFile.findSymbolByVrom` instead.
+
+### Fixed
+
+- Fix typo that prevented using `jsonify`.
+
 ## [2.6.0] - 2024-08-26
 
 ### Added
@@ -392,6 +415,7 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
 - Initial release
 
 [unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
+[2.7.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.6.0...2.7.0
 [2.6.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.1...2.6.0
 [2.5.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.0...2.5.1
 [2.5.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.4.0...2.5.0

Cargo.lock

@@ -3,7 +3,7 @@
 
 [package]
 name = "mapfile_parser"
-version = "2.6.0"
+version = "2.7.0"
 edition = "2021"
 rust-version = "1.65.0"
 authors = ["Anghelo Carvajal <[email protected]>"]

Cargo.toml

@@ -35,7 +35,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.6.0,<3.0.0
+mapfile_parser>=2.7.0,<3.0.0
 ```
 
 #### Development version
@@ -74,7 +74,7 @@ cargo add mapfile_parser
 Or add the following line manually to your `Cargo.toml` file:
 
 ```toml
-mapfile_parser = "2.6.0"
+mapfile_parser = "2.7.0"
 ```
 
 ## Versioning and changelog

README.md

@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.6.0"
+version = "2.7.0"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.8"

pyproject.toml

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 6, 0)
+__version_info__ = (2, 7, 0)
 __version__ = ".".join(map(str, __version_info__))# + "-dev0"
 __author__ = "Decompollaborate"
 

src/mapfile_parser/__init__.py

@@ -61,10 +61,12 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
             or builtRom[i + 3] != expectedRom[i + 3]
         ):
             if diffs == 0:
-                vromInfo = builtMapFile.findSymbolByVramOrVrom(i)
+                vromInfo, possibleFiles = builtMapFile.findSymbolByVrom(i)
                 extraMessage = ""
                 if vromInfo is not None:
                     extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
+                elif len(possibleFiles) > 0:
+                    extraMessage = f", in file {possibleFiles[0].asStr()}"
                 print(f"First difference at ROM addr 0x{i:X}{extraMessage}")
                 builtBytes = builtRom[i : i + 4]
                 expectedBytes = expectedRom[i : i + 4]
@@ -80,15 +82,13 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
             len(map_search_diff) < diffCount
             and builtRom[i+endian_diff] >> 2 != expectedRom[i+endian_diff] >> 2
         ):
-            vromInfo = builtMapFile.findSymbolByVramOrVrom(i)
+            vromInfo, possibleFiles = builtMapFile.findSymbolByVrom(i)
             if vromInfo is not None:
                 vromMessage = vromInfo.getAsStr()
                 if vromMessage not in map_search_diff:
                     map_search_diff.add(vromMessage)
 
-                    extraMessage = ""
-                    if vromInfo is not None:
-                        extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
+                    extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
                     print(f"Instruction difference at ROM addr 0x{i:X}{extraMessage}")
                     builtBytes = builtRom[i : i + 4]
                     expectedBytes = expectedRom[i : i + 4]
@@ -98,6 +98,17 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
                         expectedConverted = bytesConverterCallback(expectedBytes, expectedMapFile)
                         if builtConverted is not None and expectedConverted is not None:
                             print(f"{builtConverted} vs {expectedConverted}")
+            elif len(possibleFiles) > 0:
+                extraMessage = f", in file {possibleFiles[0].asStr()}"
+                print(f"Instruction difference at ROM addr 0x{i:X}{extraMessage}")
+                builtBytes = builtRom[i : i + 4]
+                expectedBytes = expectedRom[i : i + 4]
+                print(f"Bytes: {utils.hexbytes(builtBytes, addColons=addColons)} vs {utils.hexbytes(expectedBytes, addColons=addColons)}")
+                if bytesConverterCallback is not None:
+                    builtConverted = bytesConverterCallback(builtBytes, builtMapFile)
+                    expectedConverted = bytesConverterCallback(expectedBytes, expectedMapFile)
+                    if builtConverted is not None and expectedConverted is not None:
+                        print(f"{builtConverted} vs {expectedConverted}")
 
         if len(map_search_diff) >= diffCount and diffs > shift_cap:
             break

src/mapfile_parser/frontends/first_diff.py

@@ -37,7 +37,7 @@ def processArguments(args: argparse.Namespace):
     mapPath: Path = args.mapfile
     outputPath: Path|None = Path(args.output) if args.output is not None else None
     machine: bool = args.machine
-    applyFixes: bool = args.applyFixes
+    applyFixes: bool = args.apply_fixes
 
     exit(doJsonify(mapPath, outputPath, humanReadable=not machine, applyFixes=applyFixes))
 

src/mapfile_parser/frontends/jsonify.py

@@ -12,36 +12,64 @@
 from .. import utils
 
 
-def doSymInfo(mapPath: Path, symName: str) -> int:
+def doSymInfo(mapPath: Path, symName: str, *, as_vram: bool=False, as_vrom: bool=False, as_name: bool=False) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
     mapFile = mapfile.MapFile()
     mapFile.readMapFile(mapPath)
 
-    if utils.convertibleToInt(symName, 0):
-        info = mapFile.findSymbolByVramOrVrom(int(symName, 0))
+    possibleFiles: list[mapfile.File] = []
+
+    if as_vram:
+        address = int(symName, 0)
+        info, possibleFiles = mapFile.findSymbolByVram(address)
+    elif as_vrom:
+        address = int(symName, 0)
+        info, possibleFiles = mapFile.findSymbolByVrom(address)
+    elif as_name:
+        info = mapFile.findSymbolByName(symName)
+
+    # Start the guessing game
+    elif utils.convertibleToInt(symName, 0):
+        address = int(symName, 0)
+        info, possibleFiles = mapFile.findSymbolByVram(address)
+        if info is None:
+            info, possibleFiles2 = mapFile.findSymbolByVrom(address)
+            possibleFiles.extend(possibleFiles2)
     else:
         info = mapFile.findSymbolByName(symName)
 
-    if info is None:
-        print(f"'{symName}' not found in map file '{mapPath}'")
-        return 1
-    print(info.getAsStrPlusOffset(symName))
-    return 0
+    if info is not None:
+        print(info.getAsStrPlusOffset(symName))
+        return 0
+    print(f"'{symName}' not found in map file '{mapPath}'")
+    if len(possibleFiles) > 0:
+        print("But it may be a local symbol of either of the following files:")
+        for f in possibleFiles:
+            print(f"    {f.asStr()})")
+    return 1
 
 
 def processArguments(args: argparse.Namespace):
     mapPath: Path = args.mapfile
     symName: str = args.symname
+    as_vram: bool = args.vram
+    as_vrom: bool = args.vrom
+    as_name: bool = args.name
 
-    exit(doSymInfo(mapPath, symName))
+    exit(doSymInfo(mapPath, symName, as_vram=as_vram, as_vrom=as_vrom, as_name=as_name))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("sym_info", help="Display various information about a symbol or address.")
 
-    parser.add_argument("mapfile", help="Path to a map file", type=Path)
-    parser.add_argument("symname", help="symbol name or VROM/VRAM address to lookup")
+    parser.add_argument("mapfile", help="Path to a map file.", type=Path)
+    parser.add_argument("symname", help="Symbol name or VROM/VRAM address to lookup. How to treat this argument will be guessed.")
+
+    vram_vrom_group = parser.add_mutually_exclusive_group()
+    vram_vrom_group.add_argument("--vram", help="Treat the argument as a VRAM address instead of guessing.", action="store_true")
+    vram_vrom_group.add_argument("--vrom", help="Treat the argument as a VROM address instead of guessing.", action="store_true")
+    vram_vrom_group.add_argument("--name", help="Treat the argument as a symbol name instead of guessing.", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/frontends/sym_info.py

@@ -203,6 +203,7 @@ def findSymbolByName(self, symName: str) -> Symbol|None:
                 return sym
         return None
 
+    #! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
     def findSymbolByVramOrVrom(self, address: int) -> tuple[Symbol, int]|None:
         prevVram = self.vram
         prevVrom = self.vrom
@@ -244,6 +245,59 @@ def findSymbolByVramOrVrom(self, address: int) -> tuple[Symbol, int]|None:
 
         return None
 
+    def findSymbolByVram(self, address: int) -> tuple[Symbol, int]|None:
+        prevSym: Symbol|None = None
+
+        for sym in self._symbols:
+            if sym.vram == address:
+                return sym, 0
+
+            if prevSym is not None:
+                if sym.vram > address:
+                    offset = address - prevSym.vram
+                    if offset < 0:
+                        return None
+                    return prevSym, offset
+
+            prevSym = sym
+
+        if prevSym is not None:
+            if prevSym.size is not None and prevSym.vram + prevSym.size > address:
+                offset = address - prevSym.vram
+                if offset < 0:
+                    return None
+                return prevSym, offset
+
+        return None
+
+    def findSymbolByVrom(self, address: int) -> tuple[Symbol, int]|None:
+        prevVrom = self.vrom if self.vrom is not None else 0
+        prevSym: Symbol|None = None
+
+        for sym in self._symbols:
+            if sym.vrom == address:
+                return sym, 0
+
+            if prevSym is not None:
+                if sym.vrom is not None and sym.vrom > address:
+                    offset = address - prevVrom
+                    if offset < 0:
+                        return None
+                    return prevSym, offset
+
+            if sym.vrom is not None:
+                prevVrom = sym.vrom
+            prevSym = sym
+
+        if prevSym is not None:
+            if prevSym.vrom is not None and prevSym.size is not None and prevSym.vrom + prevSym.size > address:
+                offset = address - prevVrom
+                if offset < 0:
+                    return None
+                return prevSym, offset
+
+        return None
+
 
     @staticmethod
     def printCsvHeader(printVram: bool=True):
@@ -292,6 +346,9 @@ def toJson(self, humanReadable: bool=True) -> dict[str, Any]:
         fileDict["symbols"] = symbolsList
         return fileDict
 
+    def asStr(self) -> str:
+        return f"{self.filepath}({self.sectionType}) (VRAM: {self.serializeVram(True)}, VROM: {self.serializeVrom(True)}, SIZE: {self.serializeSize(humanReadable=True)})"
+
 
     def copySymbolList(self) -> list[Symbol]:
         """Returns a copy (not a reference) of the internal symbol list"""
@@ -385,6 +442,7 @@ def findSymbolByName(self, symName: str) -> FoundSymbolInfo|None:
                 return FoundSymbolInfo(file, sym)
         return None
 
+    #! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
     def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
         for file in self._filesList:
             pair = file.findSymbolByVramOrVrom(address)
@@ -393,6 +451,30 @@ def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
                 return FoundSymbolInfo(file, sym, offset)
         return None
 
+    def findSymbolByVram(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
+        possibleFiles: list[File] = []
+        for file in self._filesList:
+            pair = file.findSymbolByVram(address)
+            if pair is not None:
+                sym, offset = pair
+                return FoundSymbolInfo(file, sym, offset), []
+            if address >= file.vram and address < file.vram + file.size:
+                possibleFiles.append(file)
+        return None, possibleFiles
+
+    def findSymbolByVrom(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
+        possibleFiles: list[File] = []
+        for file in self._filesList:
+            if file.vrom is None:
+                continue
+            pair = file.findSymbolByVrom(address)
+            if pair is not None:
+                sym, offset = pair
+                return FoundSymbolInfo(file, sym, offset), []
+            if address >= file.vrom and address < file.vrom + file.size:
+                possibleFiles.append(file)
+        return None, possibleFiles
+
 
     def mixFolders(self) -> Segment:
         newSegment = Segment(self.name, self.vram, self.size, self.vrom)
@@ -622,13 +704,50 @@ def findSymbolByName(self, symName: str) -> FoundSymbolInfo|None:
                 return info
         return None
 
+    #! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
     def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
         for segment in self._segmentsList:
             info = segment.findSymbolByVramOrVrom(address)
             if info is not None:
                 return info
         return None
 
+    def findSymbolByVram(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
+        """
+        Returns a symbol with the specified VRAM address (or with an addend) if
+        it exists on the mapfile.
+
+        If no symbol if found, then a list of possible files where this symbol
+        may belong to is returned. This may happen if the symbol is not
+        globally visible.
+        """
+
+        possibleFiles: list[File] = []
+        for segment in self._segmentsList:
+            info, possibleFilesAux = segment.findSymbolByVram(address)
+            if info is not None:
+                return info, []
+            possibleFiles.extend(possibleFilesAux)
+        return None, possibleFiles
+
+    def findSymbolByVrom(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
+        """
+        Returns a symbol with the specified VRAM address (or with an addend) if
+        it exists on the mapfile.
+
+        If no symbol if found, then a list of possible files where this symbol
+        may belong to is returned. This may happen if the symbol is not
+        globally visible.
+        """
+
+        possibleFiles: list[File] = []
+        for segment in self._segmentsList:
+            info, possibleFilesAux = segment.findSymbolByVrom(address)
+            if info is not None:
+                return info, []
+            possibleFiles.extend(possibleFilesAux)
+        return None, possibleFiles
+
     def findLowestDifferingSymbol(self, otherMapFile: MapFile) -> tuple[Symbol, File, Symbol|None]|None:
         minVram = None
         found = None