Make function graph drawing not take ten thousand years (vivisect#636)

* wip

* remove

* fixes

* more

* Fix

---------

Co-authored-by: atlas0fd00m <[email protected]>

envi/memcanvas/__init__.py

@@ -150,14 +150,14 @@ def addVaText(self, text, va):
     def render(self, va, size, rend=None):
         raise Exception('Deprecated!  use renderMemory!')
 
-    def clearCanvas(self, cb=None):
+    def clearCanvas(self, cb=None, sel=None):
         if cb is not None:
             cb(None)
 
     def _beginRenderMemory(self, va, size, rend):
         pass
 
-    def _endRenderMemory(self, va, size, rend, cb=None):
+    def _endRenderMemory(self, va, size, rend, cb=None, sel=None):
         if cb is not None:
             cb(None)
 
@@ -317,7 +317,7 @@ def renderMemoryAppend(self, size, cb=None):
 
         self._endRenderAppend(cb)
 
-    def _canvasCleared(self, cb, data):
+    def _canvasCleared(self, cb, sel, data):
         va = self._canv_beginva
         maxva = self._canv_endva
         size = maxva - va
@@ -344,9 +344,11 @@ def _canvasCleared(self, cb, data):
             self.addText("\nException At %s: %s\n" % (hex(va), str(e)))
 
         # Canvas callback for render completion (or error...)
-        self._endRenderMemory(va, size, rend, cb)
+        self._endRenderMemory(va, size, rend, cb, sel)
 
-    def renderMemory(self, va, size, rend=None, cb=None):
+    def renderMemory(self, va, size, rend=None, cb=None, clear=True, sel=None):
+        if sel is None:
+            sel = '#memcanvas'
         # Set our canvas render tracking variables.
         self._canv_beginva = va
         self._canv_endva = va + size
@@ -355,10 +357,10 @@ def renderMemory(self, va, size, rend=None, cb=None):
             rend = self.currend
         self.currend = rend
 
-        clearcb = functools.partial(self._canvasCleared, cb)
+        clearcb = functools.partial(self._canvasCleared, cb, sel)
         # if this is not a "scrolled" canvas, clear it.
-        if not self._canv_scrolled:
-            self.clearCanvas(clearcb)
+        if not self._canv_scrolled and clear:
+            self.clearCanvas(clearcb, sel)
         else:
             clearcb(None)
 
@@ -373,7 +375,7 @@ def __init__(self, mem, syms=None):
         # we don't want it cleared every renderMemory call.
         self.setScrolledCanvas(True)
 
-    def clearCanvas(self, cb=None):
+    def clearCanvas(self, cb=None, sel=None):
         self.strval = ''
 
     def addText(self, text, tag=None):

envi/qt/memcanvas.py

@@ -29,14 +29,12 @@ def javaScriptConsoleMessage(self, level, msg, line, source):
 
 class VQMemoryCanvas(e_memcanvas.MemoryCanvas, QWebEngineView):
 
-    #syncSignal = QtCore.pyqtSignal()
     def __init__(self, mem, syms=None, parent=None, **kwargs):
         e_memcanvas.MemoryCanvas.__init__(self, mem=mem, syms=syms)
         QWebEngineView.__init__(self, parent=parent, **kwargs)
 
         self._canv_cache = None
         self._canv_curva = None
-        self._canv_rendtagid = '#memcanvas'
         self._canv_rend_middle = False
         self.fname = None
 
@@ -128,14 +126,14 @@ def _selectVa(self, va, cb=None):
     def _beginRenderMemory(self, va, size, rend):
         self._canv_cache = ''
 
-    def _endRenderMemory(self, va, size, rend, cb=None):
-        self._appendInside(self._canv_cache, cb)
+    def _endRenderMemory(self, va, size, rend, cb=None, sel=None):
+        self._appendInside(self._canv_cache, cb, sel)
         self._canv_cache = None
 
-    def _beginRenderVa(self, va, cb=None):
+    def _beginRenderVa(self, va, cb=None, sel=None):
         self._add_raw('<a name="viv:0x%.8x" id="a_%.8x">' % (va, va), cb)
 
-    def _endRenderVa(self, va, cb=None):
+    def _endRenderVa(self, va, cb=None, sel=None):
         self._add_raw('</a>', cb)
 
     def _beginUpdateVas(self, valist, cb=None):
@@ -184,11 +182,13 @@ def _beginRenderPrepend(self):
         self._canv_cache = ''
         self._canv_ppjump = self._canv_rendvas[0][0]
 
-    def _endRenderPrepend(self, cb=None):
+    def _endRenderPrepend(self, cb=None, sel=None):
+        if sel is None:
+            sel = '#memcanvas'
         selector = 'viv:0x%.8x' % self._canv_ppjump
         self._canv_cache = self._canv_cache.replace('`', r'\`')
         js = f'''
-        var node = document.querySelector("{self._canv_rendtagid}");
+        var node = document.querySelector("{sel}");
         node.innerHTML = `{self._canv_cache}` + node.innerHTML
 
         var snode = document.getElementsByName("{selector}");
@@ -205,11 +205,13 @@ def _endRenderPrepend(self, cb=None):
     def _beginRenderAppend(self):
         self._canv_cache = ''
 
-    def _endRenderAppend(self, cb=None):
+    def _endRenderAppend(self, cb=None, sel=None):
+        if sel is None:
+            sel = '#memcanvas'
         page = self.page()
         self._canv_cache = self._canv_cache.replace('`', r'\`')
         js = f'''
-        document.querySelector("{self._canv_rendtagid}").innerHTML += `{self._canv_cache}`;
+        document.querySelector("{sel}").innerHTML += `{self._canv_cache}`;
         '''
         self._canv_cache = None
         if cb:
@@ -243,24 +245,26 @@ def _jsSetCurVa(self, vastr):
 
     # NOTE: doing append / scroll seperately allows render to catch up
     @idlethread
-    def _appendInside(self, text, cb=None):
+    def _appendInside(self, text, cb=None, sel=None):
+        if sel is None:
+            sel = '#memcanvas'
         page = self.page()
         text = text.replace('`', r'\`')
         js = f'''
-        document.querySelector("{self._canv_rendtagid}").innerHTML += `{text}`;
+        document.querySelector("{sel}").innerHTML += `{text}`;
         '''
         if cb:
             page.runJavaScript(js, cb)
         else:
             page.runJavaScript(js)
 
-    def _add_raw(self, text, cb=None):
+    def _add_raw(self, text, cb=None, sel=None):
         # If we are in a call to renderMemory, cache til the end.
         if self._canv_cache is not None:
             self._canv_cache += text
             return
 
-        self._appendInside(text, cb)
+        self._appendInside(text, cb, sel)
 
     def addText(self, text, tag=None, cb=None):
         text = html.escape(text).encode('unicode_escape').decode('utf-8')
@@ -270,10 +274,12 @@ def addText(self, text, tag=None, cb=None):
         self._add_raw(text, cb)
 
     @idlethreadsync
-    def clearCanvas(self, cb=None):
+    def clearCanvas(self, cb=None, sel=None):
+        if sel is None:
+            sel = '#memcanvas'
         page = self.page()
         js = f'''
-        var node = document.querySelector("{self._canv_rendtagid}");
+        var node = document.querySelector("{sel}");
         if (node != null) {{
             node.innerHTML = "";
         }}

vivisect/cli.py

@@ -27,6 +27,7 @@
 
 import vdb
 
+import envi.exc as e_exc
 import envi.cli as e_cli
 import envi.common as e_common
 import envi.memory as e_memory
@@ -392,7 +393,7 @@ def do_searchopcodes(self, line):
                     self.canvas.addText('\t\t; %s (Perms: %s, Smartname: %s)' % (cmt, pname, sname))
 
                 self.canvas.addText('\n')
-            except envi.SegmentationViolation as e:
+            except e_exc.SegmentationViolation as e:
                 logger.debug("segv at 0x%x", va)
 
         self.vprint('done (%d results).' % len(res))

vivisect/qt/funcgraph.py

@@ -30,7 +30,6 @@
 
 from vqt.common import *
 from vivisect.const import *
-from envi.common import MIRE
 
 class VQVivFuncgraphCanvas(vq_memory.VivCanvasBase):
     paintUp = pyqtSignal()
@@ -98,39 +97,53 @@ def _mouseMoveEvent(self, event):
         return e_qt_memcanvas.VQMemoryCanvas.mouseMoveEvent(self, event)
 
     def _renderMemoryFinish(self, cb, data):
-        self._canv_rendtagid = '#memcanvas'
         cb(data)
 
     def _renderMemoryCallback(self, cb, data):
         if not data:
             return
-        va = int(data[0])
-        size = int(data[1])
-        self._canv_rendtagid = '#codeblock_%.8x' % va
-        # DEV: this cannot be partialmethod. It *has* to be callable
-        runner = functools.partial(self._renderMemoryFinish, cb)
-        e_memcanvas.MemoryCanvas.renderMemory(self, va, size, cb=runner)
-
-    def renderMemory(self, va, size, cb=None):
+
+        for va, size in data[:-1]:
+            va = int(va)
+            size = int(size)
+            sel = '#codeblock_%.8x' % va
+            # DEV: this cannot be partialmethod. It *has* to be callable
+            #runner = functools.partial(self._renderMemoryFinish, cb)
+            e_memcanvas.MemoryCanvas.renderMemory(self, va, size, cb=None, clear=False, sel=sel)
+
+        va, size = data[-1]
+        va = int(va)
+        size = int(size)
+        sel = '#codeblock_%.8x' % va
+        e_memcanvas.MemoryCanvas.renderMemory(self, va, size, cb=cb, clear=False, sel=sel)
+
+    def renderMemory(self, blocks, cb=None):
         '''
         Funcgraph specific renderMemory() function.
         '''
-        # For the funcgraph canvas, this will be called once per code block
         if not cb:
             cb = self.__nopcb
 
-        selector = 'codeblock_%.8x' % va
-        js = '''var node = document.querySelector("#%s");
-        if (node == null) {
-            canv = document.querySelector("#memcanvas");
-            if (canv != null) {
-                canv.innerHTML += '<div class="codeblock" id="%s"></div>'
+        js = '''var nodes = %s
+        canv = document.querySelector("#memcanvas");
+        if (canv != null) {
+            var addend = ''
+            for (const [va, size] of nodes) {
+                var selector = 'codeblock_' + va.toString(16).padStart(8, '0')
+                var node = document.getElementById("${selector}");
+                if (node == null) {
+                    var elem = document.createElement('div')
+                    elem.id = selector
+                    elem.className = 'codeblock'
+                    canv.appendChild(elem)
+                } else {
+                    node.innerHTML = ''
+                }
             }
         }
-        [%d, %d]
-        ''' % (selector, selector, va, size)
+        nodes
+        ''' % (blocks,)
         runner = functools.partial(self._renderMemoryCallback, cb)
-        logger.log(MIRE, "renderMemory(%r, %r) %r", va, cb, runner)
         self.page().runJavaScript(js, runner)
 
     def __nopcb(self):
@@ -549,7 +562,7 @@ def _refreshIfNecessary(self, va):
         Called from VWE event handlers with updated VA
         '''
         if not self._canHazAutoRefresh():
-            return 
+            return
 
         if self.vw.getFunction(va) == self.fva:
             self.refresh()
@@ -836,15 +849,20 @@ def _layoutDynadag(self, data):
 
         self.mem_canvas.page().runJavaScript(svgjs, self._layoutEdges)
 
-    def _getNodeSizes(self):
+    def _getNodeSizes(self, data):
         '''
         Actually grab all the sizes of the codeblocks that we renderd in the many calls to
         _renderCodeBlock. runJavaScript has some limited ability to return values from
         javascript land to python town, so in this case, we're shoving the offsetWidth
         and offsetHeight of each of the codeblocks into a dictionary that _layoutDynadag
         can reach into to get the sizes so it can set them for use in the line layout stuff
+
+        As with a couple other of these functions, the data param exists to make PyQt5's function
+        call ing happy. We don't do anything with it.
+        '''
+        js = '''
+        var sizes = {};
         '''
-        js = 'var sizes = {};'
 
         for nid, nprops in self.graph.getNodes():
             try:
@@ -867,27 +885,30 @@ def _renderCodeBlock(self, data):
 
         One day we'll optimize this to be one big blob of JS. But not today. But this could use
         some safety rails if the user switches functions in the middle of rendering
+
+        Side note: the data param is unused. It exists to shut PyQt5 up.
         '''
         if len(self.nodes):
             # render codeblock
-            node = self.nodes.pop(0)
-            cbva = node[1].get('cbva')
-            cbsize = node[1].get('cbsize')
-            self.mem_canvas.renderMemory(cbva, cbsize, self._renderCodeBlock)
-
-            # update _xref_cache_bg (made "real" when rendering complete)
-            endva = cbva + cbsize
-            while cbva < endva:
-                for xref in self.vw.getXrefsFrom(cbva):
-                    xrfr, xrto, xrtype, xrflag = xref
-                    self._xref_cache_bg.add(xrto)
-                    #logger.debug("adding 0x%x -> 0x%x to _xref_cache", xrfr, xrto)
-
-                lva, lsz, ltp, ltinfo = self.vw.getLocation(cbva)
-                cbva += lsz
+            blocks = []
+            for node in self.nodes:
+                cbva = node[1].get('cbva')
+                cbsize = node[1].get('cbsize')
+                blocks.append([cbva, cbsize])  # yes this has to be a list so javascript plays nicely
+
+                # update _xref_cache_bg (made "real" when rendering complete)
+                endva = cbva + cbsize
+                while cbva < endva:
+                    for xref in self.vw.getXrefsFrom(cbva):
+                        xrfr, xrto, xrtype, xrflag = xref
+                        self._xref_cache_bg.add(xrto)
+
+                    lva, lsz, ltp, ltinfo = self.vw.getLocation(cbva)
+                    cbva += lsz
+            self.mem_canvas.renderMemory(blocks, self._getNodeSizes)
 
         else:
-            self._getNodeSizes()
+            self._getNodeSizes(None)
 
     def renderFunctionGraph(self, fva=None, graph=None):
         '''