Skip to content

Commit

Permalink
Merge pull request #1220 from philborman/master
Browse files Browse the repository at this point in the history
Postprocessor optimisations, init.d default directory changed
  • Loading branch information
philborman authored Feb 13, 2018
2 parents ae14d00 + f208dac commit 8097354
Show file tree
Hide file tree
Showing 18 changed files with 74 additions and 47 deletions.
2 changes: 1 addition & 1 deletion init/lazylibrarian.default
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# the service to not start.

# [required] set path where lazylibrarian is installed:
APP_PATH=/opt/lazylibrarian
APP_PATH=/opt/LazyLibrarian

# [optional] change to 1 to enable daemon
ENABLE_DAEMON=1
Expand Down
1 change: 0 additions & 1 deletion lazylibrarian/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -882,7 +882,6 @@ def config_write(part=None):

CFG.set(section, key.lower(), value)


# sanity check for typos...
for key in list(CONFIG.keys()):
if key not in list(CONFIG_DEFINITIONS.keys()):
Expand Down
3 changes: 2 additions & 1 deletion lazylibrarian/bookwork.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from lazylibrarian.formatter import safe_unicode, plural, cleanName, unaccented, formatAuthorName, \
is_valid_booktype, check_int, getList, replace_all, makeUnicode, makeBytestr
from lib.fuzzywuzzy import fuzz
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import quote_plus, urlencode
try:
from lib.tinytag import TinyTag
Expand Down Expand Up @@ -102,7 +103,7 @@ def audioRename(bookid):
return book_filename

if check_int(total, 0) and check_int(total, 0) != cnt:
logger.warn("%s: Reported %s parts, got %i" % (exists['BookName'], b, cnt))
logger.warn("%s: Reported %s parts, got %i" % (exists['BookName'], total, cnt))
return book_filename

if '/' in track: # does the track include total (eg 1/12)
Expand Down
2 changes: 1 addition & 1 deletion lazylibrarian/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def get_cached_request(url, useCache=True, cache="XML"):

if valid_cache:
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
logger.debug("CacheHandler: Returning CACHED response for %s" % url)
logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url))
if cache == "JSON":
try:
source = json.load(open(hashfilename))
Expand Down
30 changes: 19 additions & 11 deletions lazylibrarian/calibre.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import cherrypy
import lazylibrarian
from lazylibrarian import logger, database
from lazylibrarian.formatter import unaccented, getList
from lazylibrarian.formatter import unaccented, getList, makeUnicode
from lazylibrarian.importer import addAuthorNameToDB, search_for, import_book
from lazylibrarian.librarysync import find_book_in_db
from lib.fuzzywuzzy import fuzz
Expand Down Expand Up @@ -356,29 +356,31 @@ def calibreTest():
if '(calibre ' in res and res.endswith(')'):
# extract calibredb version number
res = res.split('(calibre ')[1]
res = 'calibredb ok, version ' + res[:-1]
vernum = res[:-1]
res = 'calibredb ok, version ' + vernum
# get a list of categories and counters from the database
cats, err, rc = calibredb('list_categories', ['-i'])
cnt = 0
if rc:
if not len(cats):
res = res + '\nDatabase READ Failed'
else:
res = res + '\nDatabase: ' + err
for entry in cats.split('\n'):
try:
words = entry.split()
if words[-1].isdigit():
cnt += int(words[-1])
except IndexError:
cnt += 0
words = entry.split()
if 'ITEMS' in words:
idx = words.index('ITEMS') + 1
if words[idx].isdigit():
cnt += int(words[idx])
if cnt:
res = res + '\nDatabase READ ok'
wrt, err, rc = calibredb('add', ['--authors', 'LazyLibrarian', '--title', 'dummy', '--empty'], [])
if 'Added book ids' not in wrt:
res = res + '\nDatabase WRITE Failed'
else:
calibre_id = wrt.split("book ids: ", 1)[1].split("\n", 1)[0]
rmv, err, rc = calibredb('remove', ['--permanent', calibre_id], [])
if vernum.startswith('2'):
rmv, err, rc = calibredb('remove', [calibre_id], [])
else:
rmv, err, rc = calibredb('remove', ['--permanent', calibre_id], [])
if not rc:
res = res + '\nDatabase WRITE ok'
else:
Expand Down Expand Up @@ -421,9 +423,13 @@ def calibredb(cmd=None, prelib=None, postlib=None):
p = Popen(params, stdout=PIPE, stderr=PIPE)
res, err = p.communicate()
rc = p.returncode
res = makeUnicode(res)
err = makeUnicode(err)
if rc:
if 'Errno 111' in err:
logger.debug("calibredb returned %s: Connection refused" % rc)
elif cmd == 'list_categories' and len(res):
rc = 0 # false error return of 1 on v2.xx calibredb
else:
logger.debug("calibredb returned %s: res[%s] err[%s]" % (rc, res, err))
except Exception as e:
Expand All @@ -444,6 +450,8 @@ def calibredb(cmd=None, prelib=None, postlib=None):
try:
q = Popen(params, stdout=PIPE, stderr=PIPE)
res, err = q.communicate()
res = makeUnicode(res)
err = makeUnicode(err)
rc = q.returncode
if rc:
logger.debug("calibredb retry returned %s: res[%s] err[%s]" % (rc, res, err))
Expand Down
1 change: 1 addition & 0 deletions lazylibrarian/directparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from lazylibrarian.formatter import plural, formatAuthorName, makeUnicode
from lazylibrarian.torrentparser import url_fix
from lib.six import PY2
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import urlparse, urlencode

if PY2:
Expand Down
8 changes: 4 additions & 4 deletions lazylibrarian/formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,8 @@ def md5_utf8(txt):
def makeUnicode(txt):
# convert a bytestring to unicode, don't know what encoding it might be so try a few
# it could be a file on a windows filesystem, unix...
if txt is None:
return txt
if txt is None or not txt:
return u''
elif isinstance(txt, text_type):
return txt
for encoding in [lazylibrarian.SYS_ENCODING, 'latin-1', 'utf-8']:
Expand All @@ -248,8 +248,8 @@ def makeUnicode(txt):
def makeBytestr(txt):
# convert unicode to bytestring, needed for os.walk and os.listdir
# listdir falls over if given unicode startdir and a filename in a subdir can't be decoded to ascii
if txt is None:
return txt
if txt is None or not txt:
return b''
elif not isinstance(txt, text_type): # nothing to do if already bytestring
return txt
for encoding in [lazylibrarian.SYS_ENCODING, 'latin-1', 'utf-8']:
Expand Down
3 changes: 1 addition & 2 deletions lazylibrarian/librarysync.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from lazylibrarian.importer import update_totals, addAuthorNameToDB
from lib.fuzzywuzzy import fuzz
from lib.mobi import Mobi

# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import quote_plus, urlencode

try:
Expand Down Expand Up @@ -514,7 +514,6 @@ def LibraryScan(startdir=None, library='eBook', authid=None, remove=True):
filename = os.path.join(rootdir, files)
if PY2:
filename = filename.encode(lazylibrarian.SYS_ENCODING)
id3r = None
if TinyTag:
try:
id3r = TinyTag.get(filename)
Expand Down
2 changes: 2 additions & 0 deletions lazylibrarian/notifiers/prowl.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
from lazylibrarian import logger
from lazylibrarian.common import notifyStrings, NOTIFY_SNATCH, NOTIFY_DOWNLOAD
from lib.six import PY2
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import urlencode
# noinspection PyUnresolvedReferences
from lib.six.moves.http_client import HTTPSConnection


Expand Down
2 changes: 2 additions & 0 deletions lazylibrarian/notifiers/pushover.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@


from lib.six import PY2
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import urlencode
# noinspection PyUnresolvedReferences
from lib.six.moves.http_client import HTTPSConnection

import lazylibrarian
Expand Down
2 changes: 1 addition & 1 deletion lazylibrarian/notifiers/tweet.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import lib.oauth2 as oauth
import lib.pythontwitter as twitter

# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import parse_qsl


Expand Down
32 changes: 22 additions & 10 deletions lazylibrarian/postprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import lib.zipfile as zipfile
else:
import lib3.zipfile as zipfile

from lazylibrarian import database, logger, utorrent, transmission, qbittorrent, \
deluge, rtorrent, synology, sabnzbd, nzbget
from lazylibrarian.bookwork import audioRename, seriesInfo
Expand Down Expand Up @@ -160,11 +160,11 @@ def move_into_subdir(sourcedir, targetdir, fname, move='move'):
# move the book and any related files too, other book formats, or opf, jpg with same title
# (files begin with fname) from sourcedir to new targetdir
# can't move metadata.opf or cover.jpg or similar as can't be sure they are ours
# return how many files you moved
cnt = 0
list_dir = os.listdir(makeBytestr(sourcedir))
list_dir = [makeUnicode(item) for item in list_dir]
for ourfile in list_dir:
if int(lazylibrarian.LOGLEVEL) > 2:
logger.debug("Checking %s for %s" % (ourfile, fname))
if ourfile.startswith(fname) or is_valid_booktype(ourfile, booktype="audiobook"):
if is_valid_booktype(ourfile, booktype="book") \
or is_valid_booktype(ourfile, booktype="audiobook") \
Expand All @@ -174,12 +174,18 @@ def move_into_subdir(sourcedir, targetdir, fname, move='move'):
if lazylibrarian.CONFIG['DESTINATION_COPY'] or move == 'copy':
shutil.copyfile(os.path.join(sourcedir, ourfile), os.path.join(targetdir, ourfile))
setperm(os.path.join(targetdir, ourfile))
logger.debug("copy_into_subdir %s" % ourfile)
cnt += 1
else:
shutil.move(os.path.join(sourcedir, ourfile), os.path.join(targetdir, ourfile))
setperm(os.path.join(targetdir, ourfile))
logger.debug("move_into_subdir %s" % ourfile)
cnt += 1
except Exception as why:
logger.debug("Failed to copy/move file %s to [%s], %s %s" %
(ourfile, targetdir, type(why).__name__, str(why)))
logger.warn("Failed to copy/move file %s to [%s], %s %s" %
(ourfile, targetdir, type(why).__name__, str(why)))
continue
return cnt


def unpack_archive(pp_path, download_dir, title):
Expand Down Expand Up @@ -399,7 +405,7 @@ def processDir(reset=False):
pp_path = os.path.join(download_dir, fname)

if int(lazylibrarian.LOGLEVEL) > 2:
logger.debug("processDir %s %s" % (type(pp_path), repr(pp_path)))
logger.debug("processDir found %s %s" % (type(pp_path), repr(pp_path)))

if os.path.isfile(pp_path):
# Check for single file downloads first. Book/mag file in download root.
Expand All @@ -416,7 +422,7 @@ def processDir(reset=False):
logger.debug("Skipping %s, found a .bts file" % download_dir)
else:
aname = os.path.splitext(fname)[0]
while aname[-1] in '. ':
while aname[-1] in '_. ':
aname = aname[:-1]
targetdir = os.path.join(download_dir, aname)
if not os.path.isdir(targetdir):
Expand All @@ -430,10 +436,14 @@ def processDir(reset=False):
if os.path.isdir(targetdir):
if book['NZBmode'] in ['torrent', 'magnet', 'torznab'] and \
lazylibrarian.CONFIG['KEEP_SEEDING']:
move_into_subdir(download_dir, targetdir, fname, move='copy')
cnt = move_into_subdir(download_dir, targetdir, aname, move='copy')
else:
move_into_subdir(download_dir, targetdir, fname)
cnt = move_into_subdir(download_dir, targetdir, aname)
pp_path = targetdir
if cnt: # mark the folder to skip next time round
btsfile = os.path.join(targetdir, "%s.bts" % aname)
with open(btsfile, 'a'):
os.utime(btsfile, None)
else:
# Is file an archive, if so look inside and extract to new dir
res = unpack_archive(pp_path, download_dir, matchtitle)
Expand All @@ -442,7 +452,7 @@ def processDir(reset=False):
else:
logger.debug('Skipping unhandled file %s' % fname)

elif os.path.isdir(pp_path):
if os.path.isdir(pp_path):
logger.debug('Found folder (%s%%) [%s] for %s %s' %
(match, pp_path, book_type, matchtitle))

Expand Down Expand Up @@ -475,6 +485,8 @@ def processDir(reset=False):
skipped = True
if not skipped:
matches.append([match, pp_path, book])
if match == 100: # no point looking any further
break
else:
logger.debug('%s is not a file or a directory?' % pp_path)
else:
Expand Down
5 changes: 4 additions & 1 deletion lazylibrarian/qbittorrent.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@
from lazylibrarian import logger
from lazylibrarian.common import USER_AGENT
from lazylibrarian.formatter import check_int, getList, makeBytestr, makeUnicode

# noinspection PyUnresolvedReferences
from lib.six.moves import http_cookiejar
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import urlencode
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_request import HTTPCookieProcessor, build_opener, Request
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_error import URLError


Expand Down
1 change: 1 addition & 0 deletions lazylibrarian/torrentparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from lazylibrarian.cache import fetchURL
from lazylibrarian.formatter import plural, unaccented, makeUnicode
from lib.six import PY2, PY3, text_type
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import quote_plus, quote, urlencode, urlsplit, urlunsplit

if PY2:
Expand Down
2 changes: 1 addition & 1 deletion lazylibrarian/transmission.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ def torrentAction(method, arguments):
session_id = response.headers['x-transmission-session-id']

if not session_id:
logger.error("Expected a Session ID from Transmission")
logger.error("Expected a Session ID from Transmission, got %s" % response.status_code)
return

# Prepare next request
Expand Down
8 changes: 6 additions & 2 deletions lazylibrarian/utorrent.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
from lazylibrarian.common import USER_AGENT
from lazylibrarian.formatter import check_int, getList
from lib.six import PY2
# noinspection PyUnresolvedReferences
from lib.six.moves import http_cookiejar
from lib.six.moves.urllib_parse import quote_plus, urlencode, urljoin
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_parse import urljoin, urlencode
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_request import HTTPCookieProcessor, HTTPBasicAuthHandler, \
build_opener, install_opener, Request
# noinspection PyUnresolvedReferences
from lib.six.moves.urllib_error import HTTPError


Expand Down Expand Up @@ -144,7 +148,7 @@ def setprio(self, hashid, priority, *files):
return self._action(params)

def _action(self, params, body=None, content_type=None):
url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urllib.urlencode(params)
url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urlencode(params)
request = Request(url)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
Expand Down
8 changes: 2 additions & 6 deletions lazylibrarian/versioncheck.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@
except ImportError:
import lib.requests as requests

from lib.six import PY2

from lazylibrarian import logger, version
from lazylibrarian.common import USER_AGENT, proxyList
from lazylibrarian.formatter import check_int, makeUnicode
Expand Down Expand Up @@ -74,10 +72,8 @@ def runGit(args):
shell=True, cwd=lazylibrarian.PROG_DIR)
output, err = p.communicate()

if output:
output = makeUnicode(output).strip('\n')
if err:
err = makeUnicode(err)
output = makeUnicode(output).strip('\n')
err = makeUnicode(err)

logmsg('debug', '(RunGit)Git output: [%s]' % output)

Expand Down
9 changes: 4 additions & 5 deletions lib3/feedparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,24 +192,23 @@ def _xmlescape(data,entities={}):

# sgmllib is not available by default in Python 3; if the end user doesn't have
# it available then we'll lose illformed XML parsing and content santizing
_SGML_AVAILABLE = 0
try:
import sgmllib
_SGML_AVAILABLE = 1
except ImportError:
# This is probably Python 3, which doesn't include sgmllib anymore
try:
import lib.sgmllib3 as sgmllib
import lib3.sgmllib as sgmllib
_SGML_AVAILABLE = 1
except ImportError:
_SGML_AVAILABLE = 0

# Mock sgmllib enough to allow subclassing later on
class sgmllib(object):
class SGMLParser(object):
def goahead(self, i):
pass
def parse_starttag(self, i):
pass
else:
_SGML_AVAILABLE = 1

# sgmllib defines a number of module-level regular expressions that are
# insufficient for the XML parsing feedparser needs. Rather than modify
Expand Down

0 comments on commit 8097354

Please sign in to comment.