nameTools.py

import re

import copy
import settings
import logging
import psycopg2
import time
import urllib.parse
import threading
import os
import os.path
import unicodedata

import random
random.seed()

import pyinotify

import runStatus
import MangaCMSOld.lib.logSetup

# --------------------------------------------------------------

# Asshole scanlators who don't put their name in "[]"
# Fuck you people. Seriously
shitScanlators = ["rhs", "rh", "mri", "rhn", "se", "rhfk", "mw-rhs"]

chapVolRe     = re.compile(r"(?:(?:ch?|v(?:ol(?:ume)?)?|(?:ep)|(?:stage)|(?:pa?r?t)|(?:chapter)|(?:story)|(?:extra)|(?:load)|(?:log)) ?\d+)", re.IGNORECASE)
trailingNumRe = re.compile(r"(\d+$)", re.IGNORECASE)


# In a lot of situations, we don't have a series name (particularly for IRC downloads, etc...)
# This function tries to clean up filenames enough that we can then match the filename into
# the name database.
# It's crude as hell, but short of a neural net or something, it's as good as it's gonna get
# for fuzzy matching strings into the database.
# Something like levenshtein string distance might be interesting, but I'd be too concerned
# about false-positive matches. Failing to no-match occationally is FAR preferable to
# failing to wrong-match, so we fail no-match.
def guessSeriesFromFilename(inStr):
	inStr = inStr.lower()
	inStr = removeBrackets(inStr)

	# if there is a "." in the last 6 chars, it's probably an extension. remove it.
	if "." in inStr[-6:]:
		inStr, dummy_ext = inStr.rsplit(".", 1)

	# Strip out scanlator name strings for scanlators who are assholes and don't bracket their group name.
	for shitScanlator in shitScanlators:
		if inStr.lower().endswith(shitScanlator.lower()):
			inStr = inStr[:len(shitScanlator)*-1]

	inStr = inStr.replace("+", " ")
	inStr = inStr.replace("_", " ")
	inStr = inStr.replace("the4koma", " ")
	inStr = inStr.replace("4koma", " ")

	inStr = stripChapVol(inStr)

	inStr = inStr.strip()
	inStr = stripTrailingNumbers(inStr)

	inStr = prepFilenameForMatching(inStr)
	return inStr

def stripChapVol(inStr):
	inStr = chapVolRe.sub(" ", inStr)
	return inStr

def stripTrailingNumbers(inStr):
	inStr = trailingNumRe.sub(" ", inStr)
	return inStr

# Execution time of ~ 0.000052889607680 second (52 microseconds)
def prepFilenameForMatching(inStr):
	# inStr = cleanUnicode(inStr)
	inStr = makeFilenameSafe(inStr)
	inStr = sanitizeString(inStr)
	return inStr.lower()

def makeFilenameSafe(inStr):

	# FUCK YOU SMART-QUOTES.
	inStr = inStr.replace("“",  "\"") \
				 .replace("”",  "\"")


	offending = ['%', "	", "\n"]
	if any([tmp in inStr for tmp in offending]):
		fixed = urllib.parse.unquote(inStr)
		fixed = fixed.split("	")[0]
		fixed = fixed.split("\n")[0]
		if fixed and fixed != inStr:
			inStr = fixed

	inStr = inStr.replace("%20", " ") \
				 .replace("<",  " ") \
				 .replace(">",  " ") \
				 .replace(":",  " ") \
				 .replace("\"", " ") \
				 .replace("/",  " ") \
				 .replace("\\", " ") \
				 .replace("|",  " ") \
				 .replace("?",  " ") \
				 .replace("*",  " ") \
				 .replace('"', " ")

	# zero-width space bullshit (goddammit unicode)
	inStr = inStr.replace("\u2009",  " ") \
				 .replace("\u200A",  " ") \
				 .replace("\u200B",  " ") \
				 .replace("\u200C",  " ") \
				 .replace("\u200D",  " ") \
				 .replace("\uFEFF",  " ")

	# Collapse all the repeated spaces down.
	while inStr.find("  ")+1:
		inStr = inStr.replace("  ", " ")


	# inStr = inStr.rstrip(".")  # Windows file names can't end in dot. For some reason.
	# Fukkit, disabling. Just run on linux.

	inStr = inStr.rstrip("! ")   # Clean up trailing exclamation points
	inStr = inStr.strip(" ")    # And can't have leading or trailing spaces

	return inStr


# I have a love-hate unicode relationship. I'd /like/ to normalize everything, but doing
# so breaks more then it fixes. Arrrrgh.
def cleanUnicode(inStr):
	return unicodedata.normalize("NFKD", inStr).encode("ascii", errors="replace").decode()


bracketStripRe = re.compile(r"(\[[\+\~\-\!\d\w &:]*\])")

def removeBrackets(inStr):
	inStr = bracketStripRe.sub(" ", inStr)
	while inStr.find("  ")+1:
		inStr = inStr.replace("  ", " ")
	return inStr

# Basically used for dir-path cleaning to prep for matching, and not much else
def sanitizeString(inStr, flatten=True):
	baseName = inStr
	if flatten:
		# Adding "-" processing.
		baseName = baseName.replace("-", " ")
		baseName = baseName.replace("!", " ")

		baseName = baseName.replace("~", "")		 # Spot fixes. We'll see if they break anything
		baseName = baseName.replace(".", "")
		baseName = baseName.replace(";", "")
		baseName = baseName.replace(":", "")
		baseName = baseName.replace("-", "")
		baseName = baseName.replace("?", "")
		baseName = baseName.replace('"', "")
		baseName = baseName.replace("'", "")

	# Bracket stripping has to be done /after/ special chars are cleaned,
	# otherwise, they can break the regex.
	baseName = removeBrackets(baseName)				#clean brackets

	# baseName = baseName.replace("'", "")
	while baseName.find("  ")+1:
		baseName = baseName.replace("  ", " ")

	# baseName = unicodedata.normalize('NFKD', baseName).encode("ascii", errors="ignore")  # This will probably break shit


	return baseName.lower().strip()

def extractRating(inStr):
	# print("ExtractRating = '%s', '%s'" % (inStr, type(inStr)))
	search = re.search(r"^(.*?)\[([~+\-!]+)\](.*?)$", inStr)
	if search:
		# print("Found rating! Prefix = {pre}, rating = {rat}, postfix = {pos}".format(pre=search.group(1), rat=search.group(2), pos=search.group(3)))
		return search.group(1), search.group(2), search.group(3)
	else:
		return inStr, "", ""

def ratingStrToInt(inStr):


	pos = inStr.count("+")
	neg = inStr.count("-")

	return pos - neg

def ratingStrToFloat(inStr):

	pos = inStr.count("+")
	neg = inStr.count("-")
	half = inStr.count("~")

	return (pos - neg) + (half * 0.5)

def extractRatingToFloat(inStr):
	dummy, rating, dummy = extractRating(inStr)
	if not rating:
		return 0
	return ratingStrToFloat(rating)



def floatToRatingStr(newRating):

	# print("Rating change call!")
	newRating, remainder = int(newRating), int((newRating%1)*2)
	if newRating > 0 and newRating <= 5:
		ratingStr = "+"*newRating
	elif newRating == 0:
		ratingStr = ""
	elif newRating < 0 and newRating > -6:
		ratingStr = "-"*abs(newRating)
	else:
		raise ValueError("Invalid rating value: %s!", newRating)
	if remainder:
		ratingStr += "~"

	return ratingStr


def isProbablyImage(fileName):
	imageExtensions = [".jpeg", ".jpg", ".jpe", ".gif", ".png", ".apng", ".svg", ".bmp"]
	fileName = fileName.lower()
	for ext in imageExtensions:
		if fileName.endswith(ext):
			return True

	return False

def extractChapterVol(inStr):

	# Becuase some series have numbers in their title, we need to preferrentially
	# chose numbers preceeded by known "chapter" strings when we're looking for chapter numbers
	# and only fall back to any numbers (chpRe2) if the search-by-prefix has failed.
	chpRe1 = re.compile(r"(?<!volume)(?<!vol)(?<!v)(?<!of)(?<!season) ?(?:chapter |ch|c)(?: |_|\.)?(\d+)", re.IGNORECASE)
	chpRe2 = re.compile(r"(?<!volume)(?<!vol)(?<!v)(?<!of)(?<!season) ?(?: |_)(?: |_|\.)?(\d+)", re.IGNORECASE)
	volRe = re.compile(r"(?: |_|\-)(?:volume|vol|v|season)(?: |_|\.)?(\d+)", re.IGNORECASE)

	chap = None
	for chRe in [chpRe1, chpRe2]:
		chapF = chRe.findall(inStr)
		if chapF:
			chap  = float(chapF.pop(0)) if chapF else None
		if chap != None:
			break

	volKey = volRe.findall(inStr)
	vol    = float(volKey.pop(0))  if volKey    else None

	chap   = chap if chap != None else 0.0
	vol    = vol  if vol  != None else 0.0

	return chap, vol



# ------------------------------------------------------


# proxy that makes a DB look like a dict
# Opens a dynamically specifiable database, though the database must be one of a predefined set.
class MtNamesMapWrapper(object):

	log = logging.getLogger("Main.NSLookup")

	modes = {
		"buId->fsName" : {"cols" : ["buId", "fsSafeName"], "table" : 'munamelist',  'failOnMissing' : False},
		"buId->name"   : {"cols" : ["buId", "name"],       "table" : 'munamelist',  'failOnMissing' : False},
		"fsName->buId" : {"cols" : ["fsSafeName", "buId"], "table" : 'munamelist',  'failOnMissing' : False},
		"buId->buName" : {"cols" : ["buId", "buName"],     "table" : 'mangaseries', 'failOnMissing' : False},
		"buName->buId" : {"cols" : ["buName", "buId"],     "table" : 'mangaseries', 'failOnMissing' : False}
	}

	loaded = False
	# special class members that are picked up by the maintenance service, and used to trigger periodic updates from the DB
	# TL;DR magical runtime-introspection bullshit. Basically, if there is an
	# object defined in this file's namespace, with the `NEEDS_REFRESHING` attribute, the houskeeping task
	# will call {object}.refresh() every REFRESH_INTERVAL seconds.
	NEEDS_REFRESHING = True
	REFRESH_INTERVAL = 60*2.5

	# define conn to shut up pylinter
	conn = None
	def __init__(self, mode):


		self.updateLock = threading.Lock()


		self.log.info("Loading NSLookup")

		if not mode in self.modes:
			raise ValueError("Specified mapping mode not valid")
		self.modeKey = mode
		self.mode = self.modes[mode]
		self.openDB()

		self.lastUpdate = 0
		self.lutItems = {}

		self.queryStr = 'SELECT %s FROM %s WHERE %s=%%s;' % (self.mode["cols"][1], self.mode["table"], self.mode["cols"][0])
		self.allQueryStr = 'SELECT %s, %s FROM %s;' % (self.mode["cols"][0], self.mode["cols"][1], self.mode["table"])
		self.log.info("Mode %s, Query %s", mode, self.queryStr)
		self.log.info("Mode %s, IteratorQuery %s",  mode, self.allQueryStr)

		if runStatus.preloadDicts:
			self.loaded = True
			self.refresh()

	def stop(self):
		self.log.info("Unoading NSLookup")
		self.closeDB()

	def refresh(self):
		self.log.info("Refresh call! for %s mapping cache.", self.modeKey)
		tmp = {}
		for key, buId in self.iteritems():
			key = key.lower()

			if not key in tmp:
				tmp[key] = set([buId])
			else:
				tmp[key].add(buId)

		self.log.info("Refresh call complete. Have %s keys", len(tmp))
		self.lutItems = tmp

	def openDB(self):
		self.log.info( "NSLookup Opening DB...")
		try:
			self.conn = psycopg2.connect(dbname=settings.DATABASE_DB_NAME, user=settings.DATABASE_USER,password=settings.DATABASE_PASS)
		except psycopg2.OperationalError:
			self.conn = psycopg2.connect(host=settings.DATABASE_IP, dbname=settings.DATABASE_DB_NAME, user=settings.DATABASE_USER,password=settings.DATABASE_PASS)

		# self.conn.autocommit = True
		self.log.info("opened")

		with self.conn.cursor() as cur:
			cur.execute('''SELECT tablename FROM pg_catalog.pg_tables WHERE tablename='%s';''' % self.mode["table"])
			rets = cur.fetchall()

		self.conn.commit()
		if rets:
			rets = rets[0]
		if not self.mode["table"] in rets:   # If the DB doesn't exist, set it up.
			self.log.warning("DB Not setup for %s.", self.mode["table"])
			if self.mode['failOnMissing']:
				raise ValueError
			else:
				# We can't preload the dict, since it doesn't exist, so disable preloading.
				runStatus.preloadDicts = False

	def closeDB(self):
		self.log.info( "Closing DB...")
		self.conn.close()
		self.log.info( "done")

	def iteritems(self):


		with self.conn.cursor() as cur:
			cur.execute(self.allQueryStr)
			rets = cur.fetchall()

		self.conn.commit()

		for fsSafeName, buId in rets:
			yield fsSafeName, buId


	def __getitem__(self, key):
		if not self.loaded:
			self.loaded = True
			self.refresh()

		# if we have a key filtering function, run the key through it
		if "keyfunc" in self.mode:
			key = self.mode["keyfunc"](key)

		# db is all CITEXT, so we emulate that by calling lower on ALL THE THINGS
		key = key.lower()

		if not key in self.lutItems:
			return []

		# Have to do list comprehension so we don't return the item by reference,
		# which can lead to it getting clobbered.
		return [item for item in self.lutItems[key]]

	def __contains__(self, key):

		if key in self.lutItems[key]:
			return True
		return False




class EventHandler(pyinotify.ProcessEvent):
	def __init__(self, paths):
		super(EventHandler, self).__init__()
		self.paths = {}
		for path in paths:
			self.paths[path] = False
		self.updateLock = threading.Lock()

	def process_default(self, event):
		self.updateLock.acquire()
		# print("Dir monitor detected change!", event)
		for path in self.paths.keys():
			if event.path.startswith(path):
				self.paths[path] |= True
				# print("Changed base-path = ", path)
		# print("Event path?", event.path)
		self.updateLock.release()

	def setPathDirty(self, path):
		print("Setting path '{path}' as dirty".format(path=path))
		self.updateLock.acquire()
		self.paths[path] = True
		self.updateLock.release()

	def getClearChangedStatus(self, path):

		self.updateLock.acquire()
		ret = self.paths[path]
		self.paths[path] = False
		self.updateLock.release()

		return ret


MONITORED_FS_EVENTS = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_FROM | \
						pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB

# Caching proxy that makes a directories look like a dict.
# Does folder-name mangling to provide case-insensitivity, and provide some
# robusness to minor name variations.
class DirNameProxy(object):

	# Make it a borg class (all instances share state)
	_shared_state = {}

	log = logging.getLogger("Main.DirLookup")

	# test-mode is when the unittesting system pre-loads the dir-dict with known values,
	# so we don't have to start the dir observers (sloooow).
	# Therefore, in test-mode, we don't check if the observers exist.
	testMode = False


	def __init__(self, paths):
		self.__dict__ = self._shared_state

		self.notifierRunning = False
		self.updateLock = threading.Lock()


		self.paths = paths

		self.lastCheck = 0
		self.maxRate = 60*5
		self._dirDicts = {}


		# for watch in self.


	# special class members that are picked up by the maintenance service, and used to trigger periodic updates from the DB
	# Basically, if there is an object defined in this file's namespace, with the `NEEDS_REFRESHING` attribute, the houskeeping task
	# will call {object}.refresh() every REFRESH_INTERVAL seconds.
	# TL;DR magical runtime-introspection bullshit.
	NEEDS_REFRESHING = True
	REFRESH_INTERVAL = 60


	# define a few things to shut up pylinter
	wm       = None
	eventH   = None
	notifier = None
	def refresh(self):
		self.log.info("Refresh call! for dirMonitor system.")
		self.checkUpdate()
		self.log.info("DirMonitor system refreshed.")

	def observersActive(self):
		return self.notifierRunning

	def startDirObservers(self):
		# Observers do not need to be started for simple use, particularly
		# for quick-scripts where the filesystem is not expected to change significantly.
		# Pass useObservers=False to avoid the significant delay
		# in allocating directory observers.


		self.notifierRunning = True
		# Used to check that the directories have been loaded.
		# Should probably be broken up into `notifierRunning` and `dirsLoaded` flags.

		self.checkUpdate(force=True)
		baseDictKeys = list(self._dirDicts.keys())
		baseDictKeys.sort()

	def stop(self):
		# Only stop once (should prevent on-exit errors)
		if self.notifierRunning:
			self.log.info("Unoading DirLookup")
			self.notifierRunning = False

	def getDirDict(self, dlPath):

		self.log.info( "Loading Output Dirs for path '%s'...", dlPath)
		if not os.path.exists(dlPath):
			raise ValueError("Download path %s does not exist?" % dlPath)
		targetContents = os.listdir(dlPath)
		targetContents.sort()
		#self.log.info( "targetContents", targetContents)
		targets = {}
		for dirPath in targetContents:
			fullPath = os.path.join(dlPath, dirPath)
			if os.path.isdir(fullPath):
				baseName = getCanonicalMangaUpdatesName(dirPath)
				baseName = prepFilenameForMatching(baseName)

				if baseName in targets:
					print("ERROR - Have muliple entries for directory!")
					print("Current dir = '%s'" % fullPath)
					print("Other   dir = '%s'" % targets[baseName])
					# raise ValueError("Have muliple entries for directory!")

				targets[baseName] = fullPath

			# print("Linking '%s' to '%s'" % (fullPath, baseName))
		self.log.info( "Done")


		return targets

	def manuallyLoadDirDict(self, dirItems):
		tmp = {}
		self.testMode = True
		for name in dirItems:

			baseName = getCanonicalMangaUpdatesName(name)
			baseName = prepFilenameForMatching(baseName)
			tmp[baseName] = name

		self._dirDicts[0] = tmp


	def checkUpdate(self, force=False, skipTime=False):

		updateTime = time.time()
		if not updateTime > (self.lastCheck + self.maxRate) and (not force) and (not skipTime):
			print("DirDicts not stale!")
			return
		self.updateLock.acquire()

		self.lastCheck = updateTime

		keys = list(self.paths.keys())
		keys.sort()
		# print("Keys = ", keys)
		# print("DirNameLookup checking for changes (force=%s)!" % force)

		for key in keys:
			# Only query the filesystem at most once per *n* seconds.
			if updateTime > self.paths[key]["lastScan"] + self.paths[key]["interval"] or force or skipTime:
				self.log.info("DirLookupTool updating %s, path=%s!", key, self.paths[key]["dir"])
				self.log.info("DirLookupTool updating from Directory")
				self._dirDicts[key] = self.getDirDict(self.paths[key]["dir"])
				self.paths[key]["lastScan"] = updateTime

		self.updateLock.release()

	# Force the update of the directory containing the passed path dirPath
	# Useful for when programmatic changes are made, such as creating a directory, and
	# you want to force that change to be recognized in the dir proxy immediately.
	# This is needed because the change-watching mechanism doesn't always seem
	# to properly catch folder creation or manipulation.
	# It works great for file changes.
	def forceUpdateContainingPath(self, dirPath):

		self.updateLock.acquire()

		keys = list(self.paths.keys())
		keys.sort()
		for key in keys:
			if self.paths[key]["dir"] in dirPath:
				self.log.info("DirLookupTool updating %s, path=%s!", key, self.paths[key]["dir"])
				self.log.info("DirLookupTool updating from Directory")
				self._dirDicts[key] = self.getDirDict(self.paths[key]["dir"])
				self.paths[key]["lastScan"] = time.time()

		self.updateLock.release()

	def changeRating(self, mangaName, newRating):
		item = self[mangaName]
		if not item['fqPath']:
			raise ValueError("Invalid item")

		print("Item", item)
		print("Path", self.paths[item['sourceDict']]['dir'])
		oldPath = item['fqPath']
		self.changeRatingPath(oldPath, newRating)

	def _checkLookupNewDir(self, fromPath):
		for key in settings.ratingsSort["fromkey"]:
			if fromPath.startswith(settings.mangaFolders[key]["dir"]):
				fromBase = settings.mangaFolders[key]["dir"]
				toBase   = settings.mangaFolders[settings.ratingsSort["tokey"]]["dir"]
				print("Replacing base '%s with base '%s" % (fromBase, toBase))
				return fromPath.replace(fromBase, toBase)

		# If we don't have a directory we want to replace, we just return the string as passed
		return fromPath

	def changeRatingPath(self, oldPath, newRating):

		tmpPath = oldPath
		if hasattr(settings, "ratingsSort"):
			if newRating >= settings.ratingsSort["thresh"]:
				tmpPath = self._checkLookupNewDir(oldPath)

		prefix, dummy_rating, postfix = extractRating(tmpPath)

		if newRating == 0:
			return

		ratingStr = floatToRatingStr(newRating)

		if len(ratingStr):
			ratingStr = " [{rat}] ".format(rat=ratingStr)

		newPath = "{pre}{rat}{pos}".format(pre=prefix, rat=ratingStr, pos=postfix)
		newPath = newPath.rstrip(" ").lstrip(" ")

		# print("Oldpath = ", oldPath)
		# print("Newpath = ", newPath)
		if oldPath != newPath:
			if os.path.exists(newPath):
				raise ValueError("New path exists already!")
			else:
				os.rename(oldPath, newPath)
				print("Calling checkUpdate")
				self.checkUpdate(skipTime=True)
				print("checkUpdate Complete")


	def filterPreppedNameThroughDB(self, name):
		if not self.notifierRunning and self.testMode == False:
			self.log.warning("Directory observers not started! No directory contents will have been loaded!")
		name = getCanonicalMangaUpdatesName(name)
		name = prepFilenameForMatching(name)
		return name

	def getPathByKey(self, key):
		return self.paths[key]

	def getDirDicts(self):
		return self._dirDicts

	def getRawDirDict(self, key):
		return self._dirDicts[key]

	def getFromSpecificDict(self, dictKey, itemKey):
		filteredKey = self.filterPreppedNameThroughDB(itemKey)
		if not filteredKey:
			return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": filteredKey, "rating": None, "sourceDict": None}

		# print("ItemKey", itemKey, filteredKey)
		# print("Key = ", dictKey, filteredKey,  filteredKey in self._dirDicts[dictKey])
		if filteredKey in self._dirDicts[dictKey]:
			tmp = self._dirDicts[dictKey][filteredKey]
			return self._processItemIntoRet(tmp, itemKey, filteredKey, dictKey)

		return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": filteredKey, "rating": None, "sourceDict": None}


	def whichDictContainsKey(self, itemKey):
		baseDictKeys = list(self._dirDicts.keys())
		baseDictKeys.sort()
		for dirDictKey in baseDictKeys:
			if itemKey in self._dirDicts[dirDictKey]:
				return dirDictKey
		return False

	def iteritems(self):
		# self.checkUpdate()

		baseDictKeys = list(self._dirDicts.keys())
		baseDictKeys.sort()
		for dirDictKey in baseDictKeys:
			keys = list(self._dirDicts[dirDictKey].keys())  # I want the items sorted by name, so we have to sort the list of keys, and then iterate over that.
			keys.sort()

			for key in keys:
				item = self[key]

				# Inject the key we're iterating from, so we can see if we're fetching an item from a different/the wrong dict
				# when doing the actual lookup
				item['iterKey'] = dirDictKey
				yield key, item

	def _processItemIntoRet(self, item, origKey, filteredKey, dirDictKey):
		dummy_basePath, dirName = os.path.split(item)
		dummy_prefix, rating, dummy_postfix = extractRating(dirName)
		ret = {"fqPath" : item, "item": dirName, "inKey" : origKey, "dirKey": filteredKey, "rating": rating, "sourceDict": dirDictKey}
		return ret

	def getTotalItems(self):
		items = 0
		for item in self._dirDicts.values():
			items += len(item)
		return items

	def random(self):
		items = self.getTotalItems()

		# Special-case for no items, return nothing.
		if items == 0:
			return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": 'None', "rating": None, "sourceDict": None}
		index = random.randint(0, items-1)
		# print("Getting random item with indice", index)
		return self.getByIndex(index)

	def getByIndex(self, index):
		if index < 0 or index >= self.getTotalItems():
			raise ValueError("Index value exceeds allowable range - %s" % index)
		for dummy_key, itemSet in self._dirDicts.items():
			if index >= len(itemSet):
				index -= len(itemSet)
				continue
			else:
				item = itemSet[list(itemSet.keys())[index]]
				dummy_basePath, dirName = os.path.split(item)
				# print("Selected item with dirPath: ", item)
				filteredKey = prepFilenameForMatching(dirName)
				return self[filteredKey]
		raise ValueError("Exceeded valid range?")

	def __getitem__(self, key):
		# self.checkUpdate()
		if len(key.strip()) == 0:
			return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": key, "rating": None, "sourceDict": None}

		filteredKey = self.filterPreppedNameThroughDB(key)
		if not filteredKey:
			return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": filteredKey, "rating": None, "sourceDict": None}

		baseDictKeys = list(self._dirDicts.keys())
		baseDictKeys.sort()


		for dirDictKey in baseDictKeys:
			if filteredKey in self._dirDicts[dirDictKey]:
				tmp = self._dirDicts[dirDictKey][filteredKey]
				return self._processItemIntoRet(tmp, key, filteredKey, dirDictKey)

		return {"fqPath" : None, "item": None, "inKey" : key, "dirKey": filteredKey, "rating": None}

	def __contains__(self, key):
		# self.checkUpdate()

		key = self.filterPreppedNameThroughDB(key)

		if not key:
			return {"fqPath" : None, "item": None, "inKey" : None, "dirKey": key, "rating": None, "sourceDict": None}

		baseDictKeys = list(self._dirDicts.keys())
		baseDictKeys.sort()
		for dirDictKey in baseDictKeys:

			# Limit scanned items to < 100
			if dirDictKey > 99:
				continue

			if key in self._dirDicts[dirDictKey]:
				return key in self._dirDicts[dirDictKey]

		return False

	def __len__(self):
		ret = 0
		for dirDictKey in self._dirDicts.keys():
			ret += len(self._dirDicts[dirDictKey])
		return ret

	def is_subdir_of_paths(self, filepath):
		filepath = os.path.abspath(filepath)
		for value in self.paths.values():
			print("Checking against", value['dir'], filepath.startswith(value['dir']))
			if filepath.startswith(value['dir']):
				return True
		return False


## If we have the series name in the synonym database, look it up there, and use the ID
## to fetch the proper name from the MangaUpdates database
def getCanonicalMangaUpdatesName(sourceSeriesName):

	mId = getMangaUpdatesId(sourceSeriesName)
	canon = getCanonNameByMuId(mId)
	if canon:
		return canon
	return sourceSeriesName

muIdRegex = re.compile(r'\[MuId (\d+)\]')

## If we have the series name in the synonym database, look it up there, and use the ID
## to fetch the proper name from the MangaUpdates database
def getMangaUpdatesId(sourceSeriesName):

	# Allow the Id Override tag in the dirname to hard-code the Id.
	idS = muIdRegex.search(sourceSeriesName)
	if idS:
		return idS.group(1)

	fsName = prepFilenameForMatching(sourceSeriesName)
	if not fsName:
		return False

	mId = buIdLookup[fsName]
	if mId and len(mId) == 1:
		return mId.pop()
	return False


def getCanonNameByMuId(muId):

	if muId:
		correctSeriesName = idLookup[muId]
		if correctSeriesName and len(correctSeriesName) == 1:
			return correctSeriesName.pop()
	return None

def getAllMangaUpdatesIds(sourceSeriesName):

	fsName = prepFilenameForMatching(sourceSeriesName)
	if not fsName:
		return False

	mId = buIdLookup[fsName]
	return mId




## If we have the series name in the synonym database, look it up there, and use the ID
## to fetch the proper name from the MangaUpdates database
def haveCanonicalMangaUpdatesName(sourceSeriesName):

	mId = getMangaUpdatesId(sourceSeriesName)

	if mId:
		return True
	# mId = buIdFromName[sourceSeriesName]
	# if mId and len(mId) == 1:
	# 	return True
	return False


buIdLookup       = MtNamesMapWrapper("fsName->buId")
buSynonymsLookup = MtNamesMapWrapper("buId->name")
idLookup         = MtNamesMapWrapper("buId->buName")
buIdFromName     = MtNamesMapWrapper("buName->buId")

dirNameProxy     = DirNameProxy(settings.mangaFolders)




def testNameTools():
	import unittest


	class TestSequenceFunctions(unittest.TestCase):

		def setUp(self):
			dirNameProxy.startDirObservers()

		def test_name_001(self):
			self.assertTrue("Danshi Koukousei no Nichijou" in dirNameProxy)




	unittest.main()




if __name__ == "__main__":
	MangaCMSOld.lib.logSetup.initLogging()
	print("wat")
	# dirNameProxy.checkUpdate(force=True)
	# dirNameProxy.checkUpdate()
	# print("running")


	# names = set(["fractale", "fractale", "fractale", "fractale", "fractale", "kaze to ki no uta", "boku ni koi suru mechanical", "kaze to ki no uta", "boku ni koi suru mechanical", "magi", "k - days of blue", "k - days of blue", "gurenki - creo the crimson crises", "gurenki - creo the crimson crises", "soredemo sekai wa utsukushii", "gurenki - creo the crimson crises", "fuuka", "claymore", "himeyaka na tousaku", "himeyaka na tousaku", "himeyaka na tousaku", "kyoushi mo iroiro aru wake de", "ah my goddess", "akb49", "koroshiya ichi bangaihen", "koroshiya ichi bangaihen", "koi no okite"])
	# print (names)
	# for keyTmp, stats in dirNameProxy.iteritems():
	# 	if keyTmp in names:
	# 		print("Item in dict? ", keyTmp)

	# for nameTmp in names:
	# 	if nameTmp in dirNameProxy:
	# 		print("Have name", nameTmp)
	# 	else:
	# 		print("Do not have name", nameTmp)


	# try:
	# 	while True:
	# 		time.sleep(1)
	# 		dirNameProxy.checkUpdate()
	# except KeyboardInterrupt:
	# 	pass
	# print("Complete?")
		# print item