From 18916c1065f67619efc8c032ecfd7d0c75ffed6e Mon Sep 17 00:00:00 2001 From: Denis Kuzmenok Date: Mon, 9 Jan 2023 15:58:46 +0100 Subject: [PATCH] Initial implementation for shared library usage. --- resources/site-packages/elementum/daemon.py | 196 ++++++++---- resources/site-packages/elementum/monitor.py | 11 +- resources/site-packages/elementum/taller.py | 313 +++++++++++++++++++ 3 files changed, 461 insertions(+), 59 deletions(-) create mode 100644 resources/site-packages/elementum/taller.py diff --git a/resources/site-packages/elementum/daemon.py b/resources/site-packages/elementum/daemon.py index 6796957e6..60f46e326 100644 --- a/resources/site-packages/elementum/daemon.py +++ b/resources/site-packages/elementum/daemon.py @@ -1,3 +1,4 @@ +import io import os import errno import stat @@ -11,7 +12,9 @@ from kodi_six import xbmc, xbmcgui from six.moves import urllib_request from kodi_six.utils import py2_decode +from ctypes import c_char_p, c_int, cdll, Structure, c_longlong +from elementum.taller import follow from elementum.logger import log from elementum.osarch import PLATFORM, get_platform from elementum.config import ELEMENTUMD_HOST @@ -29,9 +32,14 @@ except: hasSubprocess = False +log_path = "" +last_exit_code = -1 binary_platform = {} repo = "elgatito/elementum-binaries" +class GoString(Structure): + _fields_ = [("p", c_char_p), ("n", c_longlong)] + def ensure_exec_perms(file_): st = os.stat(file_) os.chmod(file_, st.st_mode | stat.S_IEXEC) @@ -197,6 +205,8 @@ def jsonrpc_enabled(notify=False): return False def start_elementumd(**kwargs): + global log_path + jsonrpc_failures = 0 while jsonrpc_enabled() is False: jsonrpc_failures += 1 @@ -211,6 +221,7 @@ def start_elementumd(**kwargs): time.sleep(3) elementum_dir, elementum_binary = get_elementum_binary() + elementum_library = "elementum.so" log.info("Binary dir: %s, item: %s " % (elementum_dir, elementum_binary)) if elementum_dir is False or elementum_binary is False: @@ -224,7 +235,7 @@ def start_elementumd(**kwargs): pid = int(lf.read().rstrip(" \t\r\n\0")) if pid != os.getpid(): log.info("Killing process id %s" % pid) - os.kill(pid, 9) + os.kill(pid, 9) except OSError as e: if e.errno != 3 and e.errno != 22: # Ignore: OSError: [Errno 3] No such process @@ -235,8 +246,9 @@ def start_elementumd(**kwargs): if binary_platform["os"] == "windows": try: library_lockfile = os.path.join(py2_decode(translatePath(ADDON.getAddonInfo("profile"))), "library.db.lock") - log.warning("Removing library.db.lock file at %s ..." % library_lockfile) - os.remove(library_lockfile) + if os.path.exists(library_lockfile): + log.warning("Removing library.db.lock file at %s ..." % library_lockfile) + os.remove(library_lockfile) except Exception as e: log.error(repr(e)) @@ -244,12 +256,16 @@ def start_elementumd(**kwargs): STARTF_USESHOWWINDOW = 1 args = [elementum_binary] + shared_args = "" if ADDON.getSetting("local_port") != "": args.append("-remotePort=" + ADDON.getSetting("local_port")) + shared_args += " -remotePort=" + ADDON.getSetting("local_port") if ADDON.getSetting("remote_host") != "": args.append("-localHost=" + ADDON.getSetting("remote_host")) + shared_args += " -localHost=" + ADDON.getSetting("remote_host") if ADDON.getSetting("remote_port") != "": args.append("-localPort=" + ADDON.getSetting("remote_port")) + shared_args += " -localPort=" + ADDON.getSetting("remote_port") kwargs["cwd"] = elementum_dir if binary_platform["os"] == "windows": @@ -260,6 +276,7 @@ def start_elementumd(**kwargs): si.wShowWindow = SW_HIDE clear_fd_inherit_flags() kwargs["startupinfo"] = si + elementum_library = "elementum.dll" else: env = os.environ.copy() env["LD_LIBRARY_PATH"] = "%s:%s" % (elementum_dir, env.get("LD_LIBRARY_PATH", "")) @@ -282,9 +299,56 @@ def start_elementumd(**kwargs): log.info("elementumd: sleeping %d seconds before startup" % (delay)) time.sleep(delay) - if hasSubprocess: - return subprocess.Popen(args, **kwargs) - return False + proc = None + try: + proc = subprocess.Popen(args, **kwargs) + except: + pass + if not proc: + try: + log_dir = os.path.join(translatePath("special://temp/"), "elementum") + log_path = os.path.join(log_dir, "elementum.log") + + # Create and truncate log file before starting the library + try: + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + if not os.path.exists(log_path): + open(log_path, 'w').close() + else: + with open(log_path, 'r+') as f: + f.truncate() + except IOError as e: + log.error("Could not truncate the log: %s" % (e)) + pass + + library_path = os.path.join(elementum_dir, elementum_library) + log.info("elementumd: loading shared library from %s" % library_path) + lib = cdll.LoadLibrary(library_path) + + library_thread = threading.Thread(target=start_library, args=[lib, log_path, shared_args]) + library_thread.start() + + return library_thread + except Exception as e: + log.error("Unable to start library: %s" % e) + return None + + return proc + + +def start_library(lib, log_path, args): + global last_exit_code + + log.info("Preparing start with args '%s' and log path: %s" % (args, log_path)) + + lib.startWithLog.argtypes = [GoString, GoString] + lib.startWithLog.restype = c_int + + last_exit_code = lib.startWithLog(GoString(c_char_p(log_path.encode('utf-8')), len(log_path)), GoString(c_char_p(args.encode('utf-8')), len(args))) + log.info("Shared library execution stopped with code %d" % (last_exit_code)) + time.sleep(1) def shutdown(): try: @@ -296,7 +360,7 @@ def wait_for_abortRequested(proc, monitor): monitor.closing.wait() log.info("elementumd: exiting elementumd daemon") try: - if proc is not None: + if not isinstance(proc, threading.Thread): proc.terminate() except OSError: pass # Process already exited, nothing to terminate @@ -324,61 +388,81 @@ def elementumd_thread(monitor): continue log.info("elementumd: starting elementumd") - proc = None - if hasSubprocess: - proc = start_elementumd(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if not proc: - break - else: - log.info("elementumd: current system is unable to run the binary") - break - - threading.Thread(target=wait_for_abortRequested, args=[proc, monitor]).start() - - if not hasSubprocess: + proc = start_elementumd(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if proc is None: break - - if binary_platform["os"] == "windows": - while proc.poll() is None: - log.info(toUtf8(proc.stdout.readline().rstrip())) else: - # Kodi hangs on some Android (sigh...) systems when doing a blocking - # read. We count on the fact that Elementum daemon flushes its log - # output on \n, creating a pretty clean output - import fcntl - import select - fd = proc.stdout.fileno() - fl = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) - while proc.poll() is None: - try: - to_read, _, _ = select.select([proc.stdout], [], []) - for ro in to_read: - line = ro.readline() - if line == "": # write end is closed - break + threading.Thread(target=wait_for_abortRequested, args=[proc, monitor]).start() + + if isinstance(proc, threading.Thread): + while not os.path.exists(log_path): + time.sleep(1) + + for line in follow(io.open(log_path, 'r', encoding='utf8'), delay=0.5, selector=proc.is_alive): + try: + log.info(toUtf8(line.rstrip())) + except UnicodeDecodeError: + log.info(line.rstrip()) + if not proc.is_alive(): + break + last_code = last_exit_code + else: + if binary_platform["os"] == "windows": + while proc.poll() is None: + log.info(toUtf8(proc.stdout.readline().rstrip())) + else: + # Kodi hangs on some Android (sigh...) systems when doing a blocking + # read. We count on the fact that Elementum daemon flushes its log + # output on \n, creating a pretty clean output + import fcntl + import select + fd = proc.stdout.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + while proc.poll() is None: try: - log.info(toUtf8(line.rstrip())) - except TypeError: - pass - except IOError: - time.sleep(1) # nothing to read, sleep - - last_code = proc.returncode - if monitor_abort.abortRequested(): - break - if proc.returncode == 0 or proc.returncode == -9 or proc.returncode == -1: - continue - - if proc.returncode == 5: - restart_count = 0 - notify(getLocalizedString(30332), time=3000) - else: - restart_count += 1 - notify(getLocalizedString(30100), time=3000) + to_read, _, _ = select.select([proc.stdout], [], []) + for ro in to_read: + line = ro.readline() + if line == "": # write end is closed + break + try: + log.info(toUtf8(line.rstrip())) + except TypeError: + pass + except IOError: + time.sleep(1) # nothing to read, sleep + + if not isinstance(proc, threading.Thread): + last_code = proc.returncode + if monitor_abort.abortRequested(): + break + if proc.returncode == 0 or proc.returncode == -9 or proc.returncode == -1: + continue + + if proc.returncode == 5: + restart_count = 0 + notify(getLocalizedString(30332), time=3000) + else: + restart_count += 1 + notify(getLocalizedString(30100), time=3000) + else: + log.info("Joining library thread") + proc.join() + if monitor_abort.abortRequested(): + break + + if last_exit_code == 5: + restart_count = 0 + notify(getLocalizedString(30332), time=3000) + else: + restart_count += 1 + notify(getLocalizedString(30100), time=3000) xbmc.executebuiltin("Dialog.Close(all, true)") system_information() + + log.info("Sleeping before next restart attempt") time.sleep(5) if restart_count >= max_restart: diff --git a/resources/site-packages/elementum/monitor.py b/resources/site-packages/elementum/monitor.py index 084c49cb1..8183ac4eb 100644 --- a/resources/site-packages/elementum/monitor.py +++ b/resources/site-packages/elementum/monitor.py @@ -7,6 +7,7 @@ from six.moves import urllib_request from elementum.daemon import shutdown +from elementum.logger import log from elementum.config import init, ELEMENTUMD_HOST, ONLY_CLIENT from kodi_six.utils import PY2 @@ -26,8 +27,9 @@ def closing(self): def restart(self): try: + log.info("Triggering Elementum restart") init() - urllib_request.urlopen("%s/restart" % ELEMENTUMD_HOST) + urllib_request.urlopen("%s/restart" % ELEMENTUMD_HOST, timeout=1) except: pass @@ -41,21 +43,24 @@ def onAbortRequested(self): # Only when closing Kodi if self.abortRequested(): xbmc.executebuiltin("Dialog.Close(all, true)") + log.info("onAbortRequested") if not ONLY_CLIENT: shutdown() try: self._closing.set() self._closing.clear() except SystemExit as e: + log.info("Exit %d" % (e.code)) if e.code != 0: os._exit(0) pass def onSettingsChanged(self): try: + log.info("Triggering Elementum reload") init() - urllib_request.urlopen("%s/reload" % ELEMENTUMD_HOST) - urllib_request.urlopen("%s/cmd/clear_page_cache" % ELEMENTUMD_HOST) + urllib_request.urlopen("%s/reload" % ELEMENTUMD_HOST, timeout=1) + urllib_request.urlopen("%s/cmd/clear_page_cache" % ELEMENTUMD_HOST, timeout=1) except: pass diff --git a/resources/site-packages/elementum/taller.py b/resources/site-packages/elementum/taller.py new file mode 100644 index 000000000..099dfbb2e --- /dev/null +++ b/resources/site-packages/elementum/taller.py @@ -0,0 +1,313 @@ +import re +import sys +import time + +try: + range = xrange +except NameError: + pass + +class Tailer(object): + """\ + Implements tailing and heading functionality like GNU tail and head + commands. + """ + line_terminators = ('\r\n', '\n', '\r') + + def __init__(self, file, read_size=1024, end=False): + self.read_size = read_size + self.file = file + self.start_pos = self.file.tell() + if end: + self.seek_end() + + def splitlines(self, data): + return re.split('|'.join(self.line_terminators), data) + + def seek_end(self): + self.seek(0, 2) + + def seek(self, pos, whence=0): + self.file.seek(pos, whence) + + def read(self, read_size=None): + if read_size: + read_str = self.file.read(read_size) + else: + read_str = self.file.read() + + return len(read_str), read_str + + def seek_line_forward(self): + """\ + Searches forward from the current file position for a line terminator + and seeks to the charachter after it. + """ + pos = self.file.tell() + + bytes_read, read_str = self.read(self.read_size) + + start = 0 + if bytes_read and read_str[0] in self.line_terminators: + # The first charachter is a line terminator, don't count this one + start += 1 + + while bytes_read > 0: + # Scan forwards, counting the newlines in this bufferfull + i = start + while i < bytes_read: + if read_str[i] in self.line_terminators: + self.seek(pos + i + 1) + return self.file.tell() + i += 1 + + pos += self.read_size + self.seek(pos) + + bytes_read, read_str = self.read(self.read_size) + + return None + + def seek_line(self): + """\ + Searches backwards from the current file position for a line terminator + and seeks to the charachter after it. + """ + pos = end_pos = self.file.tell() + + read_size = self.read_size + if pos > read_size: + pos -= read_size + else: + pos = 0 + read_size = end_pos + + self.seek(pos) + + bytes_read, read_str = self.read(read_size) + + if bytes_read and read_str[-1] in self.line_terminators: + # The last charachter is a line terminator, don't count this one + bytes_read -= 1 + + if read_str[-2:] == '\r\n' and '\r\n' in self.line_terminators: + # found crlf + bytes_read -= 1 + + while bytes_read > 0: + # Scan backward, counting the newlines in this bufferfull + i = bytes_read - 1 + while i >= 0: + if read_str[i] in self.line_terminators: + self.seek(pos + i + 1) + return self.file.tell() + i -= 1 + + if pos == 0 or pos - self.read_size < 0: + # Not enought lines in the buffer, send the whole file + self.seek(0) + return None + + pos -= self.read_size + self.seek(pos) + + bytes_read, read_str = self.read(self.read_size) + + return None + + def tail(self, lines=10): + """\ + Return the last lines of the file. + """ + self.seek_end() + end_pos = self.file.tell() + + for i in range(lines): + if not self.seek_line(): + break + + data = self.file.read(end_pos - self.file.tell() - 1) + if data: + return self.splitlines(data) + else: + return [] + + def head(self, lines=10): + """\ + Return the top lines of the file. + """ + self.seek(0) + + for i in range(lines): + if not self.seek_line_forward(): + break + + end_pos = self.file.tell() + + self.seek(0) + data = self.file.read(end_pos - 1) + + if data: + return self.splitlines(data) + else: + return [] + + def follow(self, delay=1.0, selector=None): + """\ + Iterator generator that returns lines as data is added to the file. + + Based on: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/157035 + """ + trailing = True + + while 1: + where = self.file.tell() + line = self.file.readline() + if line: + if trailing and line in self.line_terminators: + # This is just the line terminator added to the end of the file + # before a new line, ignore. + trailing = False + continue + + if line[-1] in self.line_terminators: + line = line[:-1] + if line[-1:] == '\r\n' and '\r\n' in self.line_terminators: + # found crlf + line = line[:-1] + + trailing = False + yield line + elif selector is not None and not selector(): + return + else: + trailing = True + self.seek(where) + time.sleep(delay) + + def __iter__(self): + return self.follow() + + def close(self): + self.file.close() + +def tail(file, lines=10): + """\ + Return the last lines of the file. + + >>> try: + ... from StringIO import StringIO + ... except ImportError: + ... from io import StringIO + >>> f = StringIO() + >>> for i in range(11): + ... _ = f.write('Line %d\\n' % (i + 1)) + >>> tail(f, 3) + ['Line 9', 'Line 10', 'Line 11'] + """ + return Tailer(file).tail(lines) + +def head(file, lines=10): + """\ + Return the top lines of the file. + + >>> try: + ... from StringIO import StringIO + ... except ImportError: + ... from io import StringIO + >>> f = StringIO() + >>> for i in range(11): + ... _ = f.write('Line %d\\n' % (i + 1)) + >>> head(f, 3) + ['Line 1', 'Line 2', 'Line 3'] + """ + return Tailer(file).head(lines) + +def follow(file, delay=1.0, selector=None): + """\ + Iterator generator that returns lines as data is added to the file. + + >>> import os + >>> f = open('test_follow.txt', 'w') + >>> fo = open('test_follow.txt', 'r') + >>> generator = follow(fo) + >>> _ = f.write('Line 1\\n') + >>> f.flush() + >>> next(generator) + 'Line 1' + >>> _ = f.write('Line 2\\n') + >>> f.flush() + >>> next(generator) + 'Line 2' + >>> f.close() + >>> fo.close() + >>> os.remove('test_follow.txt') + """ + return Tailer(file, end=True).follow(delay, selector=selector) + +def _test(): + import doctest + doctest.testmod() + +def _main(filepath, options): + tailer = Tailer(open(filepath, 'rb')) + + try: + try: + if options.lines > 0: + if options.head: + if options.follow: + sys.stderr.write('Cannot follow from top of file.\n') + sys.exit(1) + lines = tailer.head(options.lines) + else: + lines = tailer.tail(options.lines) + + for line in lines: + print(line) + elif options.follow: + # Seek to the end so we can follow + tailer.seek_end() + + if options.follow: + for line in tailer.follow(delay=options.sleep): + print(line) + except KeyboardInterrupt: + # Escape silently + pass + finally: + tailer.close() + +def main(): + from optparse import OptionParser + import sys + + parser = OptionParser(usage='usage: %prog [options] filename') + parser.add_option('-f', '--follow', dest='follow', default=False, action='store_true', + help='output appended data as the file grows') + + parser.add_option('-n', '--lines', dest='lines', default=10, type='int', + help='output the last N lines, instead of the last 10') + + parser.add_option('-t', '--top', dest='head', default=False, action='store_true', + help='output lines from the top instead of the bottom. Does not work with follow') + + parser.add_option('-s', '--sleep-interval', dest='sleep', default=1.0, metavar='S', type='float', + help='with -f, sleep for approximately S seconds between iterations') + + parser.add_option('', '--test', dest='test', default=False, action='store_true', + help='Run some basic tests') + + (options, args) = parser.parse_args() + + if options.test: + _test() + elif not len(args) == 1: + parser.print_help() + sys.exit(1) + else: + _main(args[0], options) + + +if __name__ == '__main__': + main()