diff --git a/MANIFEST b/MANIFEST
old mode 100644
new mode 100755
index 8c8ad8c76..ea5b24785
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,16 +1,10 @@
-.mailmap
BENCHMARK.txt
Dockerfile
INSTALL.txt
LICENSE.txt
MANIFEST
README.rst
-RELEASE-0.75.0
-RELEASE-0.75.1
-RELEASE-0.75.2
-RELEASE-0.76.0.md
-RELEASE-0.77.0.md
-UPGRADE-0.75.0
+RELEASE-0.78.0.md
archive/CHANGES.txt
archive/RELEASE-0.12.0
archive/RELEASE-0.13.0
@@ -84,6 +78,11 @@ archive/RELEASE-0.62.6
archive/RELEASE-0.62.7
archive/RELEASE-0.62.8
archive/RELEASE-0.62.9
+archive/RELEASE-0.75.0
+archive/RELEASE-0.75.1
+archive/RELEASE-0.75.2
+archive/RELEASE-0.76.0.md
+archive/RELEASE-0.77.0.md
archive/UPGRADE-0.14.0
archive/UPGRADE-0.15.0
archive/UPGRADE-0.16.0
@@ -94,6 +93,7 @@ archive/UPGRADE-0.50.0
archive/UPGRADE-0.60.0
archive/UPGRADE-0.61.0
archive/UPGRADE-0.62.0
+archive/UPGRADE-0.75.0
circle.yml
docs/.gitignore
docs/autodoc/index.rst
@@ -137,7 +137,6 @@ docs/ical.rst
docs/igettext.rst
docs/index.rst
docs/ipkg.rst
-docs/log.rst
docs/odf.rst
docs/rss.rst
docs/stl.rst
@@ -164,7 +163,11 @@ itools/csv/csv_.py
itools/csv/parser.py
itools/csv/table.py
itools/database/__init__.py
-itools/database/catalog.py
+itools/database/backends/__init__.py
+itools/database/backends/catalog.py
+itools/database/backends/git.py
+itools/database/backends/lfs.py
+itools/database/backends/registry.py
itools/database/exceptions.py
itools/database/fields.py
itools/database/git.py
@@ -181,10 +184,10 @@ itools/datatypes/base.py
itools/datatypes/datetime_.py
itools/datatypes/languages.py
itools/datatypes/primitive.py
+itools/environment.json
itools/fs/__init__.py
itools/fs/common.py
itools/fs/lfs.py
-itools/fs/vfs.py
itools/gettext/__init__.py
itools/gettext/domains.py
itools/gettext/mo.py
@@ -193,7 +196,6 @@ itools/handlers/__init__.py
itools/handlers/archive.py
itools/handlers/base.py
itools/handlers/config.py
-itools/handlers/database.py
itools/handlers/file.py
itools/handlers/folder.py
itools/handlers/image.py
@@ -226,18 +228,25 @@ itools/i18n/oracle.py
itools/ical/__init__.py
itools/ical/datatypes.py
itools/ical/icalendar.py
+itools/locale/ca.mo
itools/locale/ca.po
+itools/locale/de.mo
itools/locale/de.po
+itools/locale/en.mo
itools/locale/en.po
+itools/locale/es.mo
itools/locale/es.po
+itools/locale/fr.mo
itools/locale/fr.po
+itools/locale/it.mo
itools/locale/it.po
itools/locale/locale.pot
+itools/locale/nl.mo
itools/locale/nl.po
+itools/locale/nl_BE.mo
itools/locale/nl_BE.po
+itools/locale/zh.mo
itools/locale/zh.po
-itools/log/__init__.py
-itools/log/log.py
itools/loop/__init__.py
itools/loop/loop.py
itools/odf/OpenDocument-v1.2-cd05-schema.rng
@@ -265,6 +274,7 @@ itools/pkg/build.py
itools/pkg/build_gulp.py
itools/pkg/git.py
itools/pkg/handlers.py
+itools/pkg/update_locale.py
itools/pkg/utils.py
itools/python/__init__.py
itools/python/python.py
@@ -304,7 +314,6 @@ itools/web/exceptions.py
itools/web/headers.py
itools/web/messages.py
itools/web/router.py
-itools/web/server.py
itools/web/static.py
itools/web/utils.py
itools/web/views.py
@@ -336,7 +345,6 @@ scripts/idb-inspect.py
scripts/igettext-build.py
scripts/igettext-extract.py
scripts/igettext-merge.py
-scripts/iodf-greek.py
scripts/ipkg-docs.py
scripts/ipkg-quality.py
scripts/ipkg-update-locale.py
@@ -412,7 +420,14 @@ test/test_xml.py
test/test_xmlfile.py
test/tests/gettext_en_es.xlf
test/tests/hello.txt
+test/tests/index.html.ca
+test/tests/index.html.de
test/tests/index.html.en
+test/tests/index.html.es
+test/tests/index.html.fr
+test/tests/index.html.nl
+test/tests/index.html.nl_BE
+test/tests/index.html.zh
test/tests/localizermsgs.tmx
test/tests/sample-rss-2.xml
test/tests/test.csv
diff --git a/README.rst b/README.rst
index 063fd1bd1..638aef742 100644
--- a/README.rst
+++ b/README.rst
@@ -6,16 +6,16 @@ meta-package for easier development and deployment.
The packages included are::
- itools itools.ical itools.srx
- itools.core itools.log itools.stl
- itools.csv itools.loop itools.tmx
- itools.database itools.odf itools.uri
- itools.datatypes itools.office itools.web
- itools.fs itools.pdf itools.workflow
- itools.gettext itools.pkg itools.xliff
- itools.handlers itools.python itools.xml
- itools.html itools.relaxng itools.xmlfile
- itools.i18n itools.rss
+ itools itools.ical itools.stl
+ itools.core itools.loop itools.tmx
+ itools.csv itools.odf itools.uri
+ itools.database itools.office itools.validators
+ itools.datatypes itools.pdf itools.web
+ itools.fs itools.pkg itools.workflow
+ itools.gettext itools.python itools.xliff
+ itools.handlers itools.relaxng itools.xml
+ itools.html itools.rss itools.xmlfile
+ itools.i18n itools.srx
The scripts included are::
diff --git a/RELEASE-0.78.0.md b/RELEASE-0.78.0.md
old mode 100644
new mode 100755
diff --git a/docs/conf.py b/docs/conf.py
index 2304a350e..65b139dda 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -269,7 +269,7 @@ def setup(app):
synopsis = "Itools %s module" % module
# And the save the file
- print '[autodoc] make the modules/%s.rst file' % module
+ print('[autodoc] make the modules/%s.rst file' % module)
with open('autodoc/modules/%s.rst' % module, 'w') as rst_file:
rst_file.write(template_module.format(module=module,
synopsis=synopsis))
diff --git a/docs/examples/csv/clients.py b/docs/examples/csv/clients.py
index f3151c59b..1bad2bb9f 100644
--- a/docs/examples/csv/clients.py
+++ b/docs/examples/csv/clients.py
@@ -19,7 +19,7 @@
# Import from itools
from itools.csv import CSVFile
-from itools.handlers import RWDatabase
+from itools.database import RWDatabase
from itools.datatypes import Integer, Unicode, String, Date
@@ -35,7 +35,7 @@ class Clients(CSVFile):
if __name__ == '__main__':
- rw_database = RWDatabase()
+ rw_database = RWDatabase(path="docs/examples/csv/", size_min=2048, size_max=4096, backend='lfs')
clients = rw_database.get_handler("clients.csv", Clients)
# Access a column by its name
@@ -48,4 +48,4 @@ class Clients(CSVFile):
clients.add_row(
[250, u'J. David Ibanez', 'jdavid@itaapy.com', date(2007, 1, 1)])
- print clients.to_str()
+ print(clients.to_str())
diff --git a/docs/examples/gettext/hello.py b/docs/examples/gettext/hello.py
index 4d7c748ad..76235bd00 100644
--- a/docs/examples/gettext/hello.py
+++ b/docs/examples/gettext/hello.py
@@ -12,7 +12,7 @@
def say_hello():
message = MSG(u'Hello World')
- print message.gettext()
+ print(message.gettext())
def get_template(name):
@@ -34,7 +34,7 @@ def get_template(name):
def tell_fable():
template = get_template('fable.xhtml')
- print open(template).read()
+ print(open(template).read())
say_hello()
diff --git a/docs/examples/html/test_parser.py b/docs/examples/html/test_parser.py
index f99b9605b..fb80ce139 100644
--- a/docs/examples/html/test_parser.py
+++ b/docs/examples/html/test_parser.py
@@ -6,13 +6,13 @@
data = open('hello.html').read()
for type, value, line in HTMLParser(data):
if type == DOCUMENT_TYPE:
- print 'DOC TYPE :', repr(value)
+ print("DOC TYPE : {}".format(repr(value)))
elif type == START_ELEMENT:
tag_uri, tag_name, attributes = value
- print 'START TAG :', tag_name
+ print("START TAG : {}".format(tag_name))
elif type == END_ELEMENT:
tag_uri, tag_name = value
- print 'END TAG :', tag_name
+ print("END TAG : {}".format(tag_name))
elif type == TEXT:
- print 'TEXT :', value
+ print("TEXT {} :".format(value))
diff --git a/docs/examples/web/cal.py b/docs/examples/web/cal.py
index 0a0046f03..a02721763 100644
--- a/docs/examples/web/cal.py
+++ b/docs/examples/web/cal.py
@@ -23,7 +23,7 @@
import datetime
# Import from itools
-from itools.handlers import RWDatabase
+from itools.database import RWDatabase
from itools.loop import Loop
from itools.uri import get_reference
from itools.web import WebServer, RootResource, Resource, BaseView
diff --git a/docs/examples/xml/test_parser.py b/docs/examples/xml/test_parser.py
index 984b79052..382ca6e31 100644
--- a/docs/examples/xml/test_parser.py
+++ b/docs/examples/xml/test_parser.py
@@ -13,10 +13,10 @@
for type, value, line in XMLParser(data):
if type == START_ELEMENT:
tag_uri, tag_name, attributes = value
- print 'START TAG :', tag_name, attributes
+ print("START TAG : {} {}".format(tag_name, attributes))
elif type == END_ELEMENT:
tag_uri, tag_name = value
- print 'END TAG :', tag_name
+ print("END TAG : {}".format(tag_name))
elif type == TEXT:
- print 'TEXT :', value
+ print("TEXT {} :".format(value))
diff --git a/docs/log.rst b/docs/log.rst
deleted file mode 100644
index 5f41bd007..000000000
--- a/docs/log.rst
+++ /dev/null
@@ -1,173 +0,0 @@
-:mod:`itools.log` -- Logging
-****************************
-
-.. module:: itools.log
- :synopsis: Logging
-
-.. index::
- single: Logging
-
-.. contents::
-
-The :mod:`itools.log` package provides a simple programming interface for
-logging events (errors, warning messages, etc.). It is inspired by the
-logging facilities included in the `GLib
-`_ library.
-
-Levels
-======
-
-Every log event belongs to one of these five levels:
-
-.. data:: FATAL
-
- Log level for fatal errors, see :func:`log_fatal`
-
-.. data:: ERROR
-
- Log level for common errors, see :func:`log_error`
-
-.. data:: WARNING
-
- Log level for warning messages, see :func:`log_warning`
-
-.. data:: INFO
-
- Log level for informational messages, see :func:`log_info`
-
-.. data:: DEBUG
-
- Log level for debug messages, see :func:`log_debug`
-
-
-Logging functions
-=================
-
-For every level there is a function. Below we define the default behavior
-of these functions, we will see later how to override this behavior.
-
-.. function:: log_fatal(message, domain=None)
-
- Prints the given message into the standard error, and then aborts the
- application.
-
-.. function:: log_error(message, domain=None)
-
- Prints the given message into the standard error.
-
-.. function:: log_warning(message, domain=None)
-
- Prints the given message into the standard error.
-
-.. function:: log_info(message, domain=None)
-
- Prints the given message into the standard output.
-
-.. function:: log_debug(message, domain=None)
-
- By default this function does nothing, debug messages are ignored.
-
-The ``domain`` argument allows to classify the log events by application
-domains. This argument is optional, if not given then the event belongs to
-the default domain.
-
-.. note::
-
- Through :mod:`itools` we define one domain per package (``itools.http``,
- ``itools.web``, etc.)
-
-Here there are some examples:
-
-.. code-block:: python
-
- >>> from itools.log import log_fatal, log_error, log_warning, log_debug
- >>> log_error('Internal Server Error', domain='itools.http')
- 2009-08-21 15:06:22 tucu itools.http[7268]: Internal Server Error
- >>> log_debug('I am here')
- >>> log_warning('Failed to connect to SMTP host', domain='itools.mail')
- 2009-08-21 15:07:23 tucu itools.mail[7268]: Failed to connect to SMTP host
- >>> log_fatal('Panic')
- 2009-08-21 15:07:39 tucu [7268]: Panic
-
-It can be appreciated that the format of the log line looks a lot like the
-syslog messages of Unix systems; except for the date, which is in a different
-format.
-
-More important is the fact that the itools logging system allows log events to
-span multiple lines. For instance, by default, if we are handling an
-exception while logging, the traceback will be printed:
-
-.. code-block:: python
-
- >>> try:
- ... 5/0
- ... except Exception:
- ... log_error('Division failed')
- ...
- 2009-08-21 15:16:53 tucu [7362]: Division failed
- Traceback (most recent call last):
- File "", line 2, in
- ZeroDivisionError: integer division or modulo by zero
-
-This allows to recover from errors while recording them.
-
-
-Override the default behavior
-=============================
-
-To override the default behavior at least one new logger must be registered,
-this is done with the :func:`register_logger` function:
-
-
-.. function:: register_logger(logger, \*domains)
-
- Register the given logger object for the given domains.
-
-For instance:
-
-.. code-block:: python
-
- from itools.log import Logger, WARNING, register_logger
-
- logger = Logger('/tmp/log', WARNING)
- register_logger(logger, None)
-
-With the code above errors and warning messages will be written to the
-``/tmp/log`` file, while debug and informational messages will be ignored.
-This will become the default behavior for all domains.
-
-Here there is the description of the default logger class:
-
-.. class:: Logger(log_file=None, min_level=INFO)
-
- By default messages are printed to the standard error or the standard
- output, depending on the level of the message. If the ``log_file``
- argument is given, it must be a file path, then messages will be written
- to the indicated file instead of printed.
-
- By default debug messages are ignored. The argument ``min_level`` allows
- to change this, for instance, to log all messages, pass the :data:`DEBUG`
- value.
-
- .. method:: format_header(domain, level, message)
-
- TODO
-
- .. method:: get_body()
-
- TODO
-
- .. method:: format_body()
-
- TODO
-
- .. method:: format(domain, level, message)
-
- TODO
-
- .. method:: log(domain, level, message)
-
- TODO
-
-It is possible to subclass the :class:`Logger` class to personalize the
-behavior of the logger as needed.
diff --git a/itools/__init__.py b/itools/__init__.py
index e347a13e2..5c824df17 100644
--- a/itools/__init__.py
+++ b/itools/__init__.py
@@ -14,9 +14,16 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+# Import from standard lib
+from logging import getLogger, NullHandler
+
# Import from itools
-from core import get_version
+from .core import get_version
+getLogger("itools.core").addHandler(NullHandler())
+getLogger("itools.web").addHandler(NullHandler())
+getLogger("itools.database").addHandler(NullHandler())
+getLogger("itools.stl").addHandler(NullHandler())
+getLogger("itools.catalog").addHandler(NullHandler())
__version__ = get_version()
-
diff --git a/itools/core/__init__.py b/itools/core/__init__.py
index 936bcb012..97b2358a8 100644
--- a/itools/core/__init__.py
+++ b/itools/core/__init__.py
@@ -20,23 +20,22 @@
from sys import platform
# Import from itools
-from cache import LRUCache
-from freeze import freeze, frozendict, frozenlist
-from lazy import lazy
-from mimetypes_ import add_type, guess_all_extensions, guess_extension
-from mimetypes_ import guess_type, has_encoding, has_extension
-from odict import OrderedDict
-from prototypes import prototype_type, prototype, is_prototype
-from prototypes import proto_property, proto_lazy_property
-from timezones import fixed_offset, local_tz
-from utils import get_abspath, merge_dicts, get_sizeof, get_pipe, get_version
+from .cache import LRUCache
+from .freeze import freeze, frozendict, frozenlist
+from .lazy import lazy
+from .mimetypes_ import add_type, guess_all_extensions, guess_extension
+from .mimetypes_ import guess_type, has_encoding, has_extension
+from .odict import OrderedDict
+from .prototypes import prototype_type, prototype, is_prototype
+from .prototypes import proto_property, proto_lazy_property
+from .timezones import fixed_offset, local_tz
+from .utils import get_abspath, merge_dicts, get_sizeof, get_pipe, get_version
if platform[:3] == 'win':
- from _win import become_daemon, fork, get_time_spent, vmsize
+ from ._win import become_daemon, fork, get_time_spent, vmsize
else:
- from _unix import become_daemon, fork, get_time_spent, vmsize
-
+ from ._unix import become_daemon, fork, get_time_spent, vmsize
__all__ = [
diff --git a/itools/core/_unix.py b/itools/core/_unix.py
index 5a006fe62..803e18b36 100644
--- a/itools/core/_unix.py
+++ b/itools/core/_unix.py
@@ -21,7 +21,6 @@
from sys import exit, stdin, stdout, stderr
-
def vmsize():
"""Returns the resident size in bytes.
"""
@@ -51,7 +50,7 @@ def become_daemon():
try:
pid = fork()
except OSError:
- print 'unable to fork'
+ print('unable to fork')
exit(1)
if pid == 0:
@@ -67,4 +66,3 @@ def become_daemon():
dup2(file_desc, 2)
else:
exit()
-
diff --git a/itools/core/_win.py b/itools/core/_win.py
index 0cf6ff20d..7fca8089e 100644
--- a/itools/core/_win.py
+++ b/itools/core/_win.py
@@ -23,8 +23,7 @@
from win32con import PROCESS_QUERY_INFORMATION, PROCESS_VM_READ
from win32process import GetProcessTimes, GetProcessMemoryInfo
except ImportError:
- print 'Warning: win32 is not installed'
-
+ print('Warning: win32 is not installed')
def get_win32_handle(pid):
@@ -52,19 +51,16 @@ def get_time_spent(mode='both', since=0.0):
return (data['KernelTime'] + data['UserTime']) / 10000000.0 - since
-
def vmsize():
handle = get_win32_handle(getpid())
m = GetProcessMemoryInfo(handle)
return m["WorkingSetSize"]
-
def become_daemon():
- raise NotImplementedError, 'become_daemon not yet implemented for Windows'
-
+ raise NotImplementedError('become_daemon not yet implemented for Windows')
def fork():
- raise NotImplementedError, 'fork not yet implemented for Windows'
+ raise NotImplementedError('fork not yet implemented for Windows')
diff --git a/itools/core/cache.py b/itools/core/cache.py
index 49993c365..8f448c886 100644
--- a/itools/core/cache.py
+++ b/itools/core/cache.py
@@ -21,7 +21,7 @@
"""
# Import from itools
-from odict import OrderedDict
+from .odict import OrderedDict
class LRUCache(OrderedDict):
@@ -63,18 +63,18 @@ def __init__(self, size_min, size_max=None, automatic=True):
# Check arguments type
if type(size_min) is not int:
error = "the 'size_min' argument must be an int, not '%s'"
- raise TypeError, error % type(size_min)
+ raise TypeError(error % type(size_min))
if type(automatic) is not bool:
error = "the 'automatic' argument must be an int, not '%s'"
- raise TypeError, error % type(automatic)
+ raise TypeError(error % type(automatic))
if size_max is None:
size_max = size_min
elif type(size_max) is not int:
error = "the 'size_max' argument must be an int, not '%s'"
- raise TypeError, error % type(size_max)
+ raise TypeError(error % type(size_max))
elif size_max < size_min:
- raise ValueError, "the 'size_max' is smaller than 'size_min'"
+ raise ValueError("the 'size_max' is smaller than 'size_min'")
# Initialize the dict
super(LRUCache, self).__init__()
@@ -84,7 +84,6 @@ def __init__(self, size_min, size_max=None, automatic=True):
# Whether to free memory automatically or not (boolean)
self.automatic = automatic
-
def _append(self, key):
super(LRUCache, self)._append(key)
@@ -93,7 +92,6 @@ def _append(self, key):
while len(self) > self.size_min:
self.popitem()
-
def touch(self, key):
# (1) Get the node from the key-to-node map
node = self.key2node[key]
@@ -115,4 +113,3 @@ def touch(self, key):
node.next = None
self.last.next = node
self.last = node
-
diff --git a/itools/core/freeze.py b/itools/core/freeze.py
index 316be11c8..3df572302 100644
--- a/itools/core/freeze.py
+++ b/itools/core/freeze.py
@@ -15,65 +15,50 @@
# along with this program. If not, see .
-
class frozenlist(list):
__slots__ = []
-
#######################################################################
# Mutable operations must raise 'TypeError'
def __delitem__(self, index):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def __delslice__(self, left, right):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def __iadd__(self, alist):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def __imul__(self, alist):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def __setitem__(self, index, value):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def __setslice__(self, left, right, value):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def append(self, item):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def extend(self, alist):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def insert(self, index, value):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def pop(self, index=-1):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def remove(self, value):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def reverse(self):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
def sort(self, cmp=None, key=None, reverse=False):
- raise TypeError, 'frozenlists are not mutable'
-
+ raise TypeError('frozenlists are not mutable')
#######################################################################
# Non-mutable operations
@@ -81,75 +66,60 @@ def __add__(self, alist):
alist = list(self) + alist
return frozenlist(alist)
-
def __hash__(self):
# TODO Implement frozenlists hash-ability
- raise NotImplementedError, 'frozenlists not yet hashable'
-
+ raise NotImplementedError('frozenlists not yet hashable')
def __mul__(self, factor):
alist = list(self) * factor
return frozenlist(alist)
-
def __rmul__(self, factor):
alist = list(self) * factor
return frozenlist(alist)
-
def __repr__(self):
- return 'frozenlist([%s])' % ', '.join([ repr(x) for x in self ])
-
+ return 'frozenlist([%s])' % ', '.join([repr(x) for x in self])
class frozendict(dict):
__slots__ = []
-
#######################################################################
# Mutable operations must raise 'TypeError'
def __delitem__(self, index):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def __setitem__(self, key, value):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def clear(self):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def pop(self, key, default=None):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def popitem(self):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def setdefault(self, key, default=None):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
def update(self, a_dict=None, **kw):
- raise TypeError, 'frozendicts are not mutable'
-
+ raise TypeError('frozendicts are not mutable')
#######################################################################
# Non-mutable operations
def __hash__(self):
# TODO Implement frozendicts hash-ability
- raise NotImplementedError, 'frozendicts not yet hashable'
-
+ raise NotImplementedError('frozendicts not yet hashable')
def __repr__(self):
- aux = [ "%s: %s" % (repr(k), repr(v)) for k, v in self.items() ]
+ aux = ["%s: %s" % (repr(k), repr(v)) for k, v in self.items()]
return 'frozendict({%s})' % ', '.join(aux)
-
def freeze(value):
# Freeze
value_type = type(value)
@@ -163,6 +133,4 @@ def freeze(value):
if isinstance(value, (frozenlist, frozendict, frozenset)):
return value
# Error
- raise TypeError, 'unable to freeze "%s"' % value_type
-
-
+ raise TypeError('unable to freeze "%s"' % value_type)
diff --git a/itools/core/lazy.py b/itools/core/lazy.py
index 448c53481..8bc67ed6d 100644
--- a/itools/core/lazy.py
+++ b/itools/core/lazy.py
@@ -15,7 +15,6 @@
# along with this program. If not, see .
-
# From http://blog.pythonisito.com/2008/08/lazy-descriptors.html
class lazy(object):
@@ -25,16 +24,13 @@ def __init__(self, meth):
self.__name__ = meth.__name__
self.__doc__ = meth.__doc__
-
def __get__(self, instance, owner):
if instance is None:
return self
- name = self.meth.func_name
+ name = self.meth.__name__
value = self.meth(instance)
instance.__dict__[name] = value
return value
-
def __repr__(self):
return '%s wrapps %s' % (object.__repr__(self), repr(self.meth))
-
diff --git a/itools/core/odict.py b/itools/core/odict.py
old mode 100644
new mode 100755
index 42dffe648..77830949f
--- a/itools/core/odict.py
+++ b/itools/core/odict.py
@@ -24,15 +24,14 @@ class DNode(object):
__slots__ = ['prev', 'next', 'key']
-
def __init__(self, key):
self.key = key
-
class OrderedDict(dict):
def __init__(self, items=None):
+ super().__init__()
# The doubly-linked list
self.first = None
self.last = None
@@ -43,19 +42,18 @@ def __init__(self, items=None):
for key, value in items:
self[key] = value
-
def _check_integrity(self):
"""This method is for testing purposes, it checks the internal
data structures are consistent.
"""
keys = self.keys()
- keys.sort()
+ keys = sorted(list(keys))
# Check the key-to-node mapping
keys2 = self.key2node.keys()
- keys2.sort()
+ keys2 = sorted(list(keys2))
assert keys == keys2
# Check the key-to-node against the doubly-linked list
- for key, node in self.key2node.iteritems():
+ for key, node in self.key2node.items():
assert type(key) is type(node.key)
assert key == node.key
# Check the doubly-linked list against the cache
@@ -67,7 +65,6 @@ def _check_integrity(self):
node = node.next
assert len(keys) == 0
-
def _append(self, key):
node = DNode(key)
@@ -83,7 +80,6 @@ def _append(self, key):
self.last.next = node
self.last = node
-
def _remove(self, key):
# (1) Pop the node from the key-to-node map
node = self.key2node.pop(key)
@@ -99,7 +95,6 @@ def _remove(self, key):
else:
node.next.prev = node.prev
-
######################################################################
# Override dict API
def __iter__(self):
@@ -108,82 +103,67 @@ def __iter__(self):
yield node.key
node = node.next
-
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
self._append(key)
-
def __delitem__(self, key):
self._remove(key)
dict.__delitem__(self, key)
-
def clear(self):
dict.clear(self)
self.key2node.clear()
self.first = self.last = None
-
def copy(self):
message = "use 'copy.deepcopy' to copy an ordered dict"
- raise NotImplementedError, message
-
+ raise NotImplementedError(message)
def fromkeys(self, seq, value=None):
- raise NotImplementedError, "the 'fromkeys' method is not supported"
-
+ raise NotImplementedError("the 'fromkeys' method is not supported")
def items(self):
return list(self.iteritems())
-
def iteritems(self):
node = self.first
while node is not None:
yield node.key, self[node.key]
node = node.next
-
def iterkeys(self):
node = self.first
while node is not None:
yield node.key
node = node.next
-
def itervalues(self):
node = self.first
while node is not None:
yield self[node.key]
node = node.next
-
def keys(self):
return list(self.iterkeys())
-
def pop(self, key):
self._remove(key)
return dict.pop(self, key)
-
def popitem(self):
if self.first is None:
- raise KeyError, 'popitem(): ordered dict is empty'
+ raise KeyError('popitem(): ordered dict is empty')
key = self.first.key
value = self[key]
del self[key]
return (key, value)
-
def setdefault(self, key, default=None):
- raise NotImplementedError, "the 'setdefault' method is not supported"
-
+ raise NotImplementedError("the 'setdefault' method is not supported")
def update(self, value=None, **kw):
- raise NotImplementedError, "the 'update' method is not supported"
-
+ raise NotImplementedError("the 'update' method is not supported")
def values(self):
return list(self.itervalues())
diff --git a/itools/core/prototypes.py b/itools/core/prototypes.py
old mode 100644
new mode 100755
index 35f739d8c..13148f210
--- a/itools/core/prototypes.py
+++ b/itools/core/prototypes.py
@@ -15,15 +15,16 @@
# along with this program. If not, see .
# Import from the Standard Library
+import traceback
+from logging import getLogger
from sys import _getframe
from types import FunctionType
-# Import from itools
-from itools.log import log_error
-
# Import from here
-from lazy import lazy
+from .lazy import lazy
+
+log = getLogger("itools.core")
"""
This module provides prototype-based programming:
@@ -53,7 +54,6 @@ class proto2(proto1):
"""
-
class prototype_type(type):
def __new__(mcs, class_name, bases, dict):
@@ -107,22 +107,17 @@ class A(prototype):
del dict[source_name]
# Fix the name
if type(value) is classmethod:
- value.__get__(None, dict).im_func.__name__ = name
+ value.__get__(None, dict).__func__.__name__ = name
elif type(value) is proto_property:
value.__name__ = name
elif type(value) is proto_lazy_property:
value.__name__ = name
-
# Make and return the class
return type.__new__(mcs, class_name, bases, dict)
-
-class prototype(object):
-
- __metaclass__ = prototype_type
-
+class prototype(object, metaclass=prototype_type):
def __new__(cls, *args, **kw):
"""
@@ -140,48 +135,44 @@ def __new__(cls, *args, **kw):
# Ok
return new_class
-
def __init__(self, *args, **kw):
pass
-
class proto_property(lazy):
def __get__(self, instance, owner):
try:
value = self.meth(owner)
except Exception as e:
- msg = 'Error on proto property:\n'
- log_error(msg + str(e), domain='itools.core')
+ tb = traceback.format_exc()
+ log.error("Error on proto property: {}".format(tb), exc_info=True)
raise
return value
-
class proto_lazy_property(lazy):
def __get__(self, instance, owner):
name = self.__name__
for cls in owner.__mro__:
if name in cls.__dict__:
- name = self.meth.func_name
+ name = self.meth.__name__
try:
value = self.meth(owner)
except Exception as e:
- msg = 'Error on proto lazy property:\n'
- log_error(msg + str(e), domain='itools.core')
+ tb = traceback.format_exc()
+ log.error("Error on proto lazy property: {}".format(tb), exc_info=True)
raise
setattr(owner, name, value)
return value
-
def is_prototype(value, cls):
from itools.gettext import MSG
from itools.web import INFO, ERROR
for c in [MSG, INFO, ERROR]:
if cls is c and isinstance(value, c):
- print("Warning: is_prototype(xxx, MSG) is obsolete. MSG is not a prototype anymore")
+ log.debug("Warning: is_prototype(xxx, MSG) is obsolete. MSG is not a prototype anymore")
return True
return issubclass(type(value), prototype_type) and issubclass(value, cls)
diff --git a/itools/core/timezones.py b/itools/core/timezones.py
old mode 100644
new mode 100755
index 5ad907a07..f8287b7d0
--- a/itools/core/timezones.py
+++ b/itools/core/timezones.py
@@ -73,37 +73,34 @@ def fixed_offset(offset, _tzinfos={}):
###########################################################################
# Local Time (copied from from Python datetime doc)
###########################################################################
-ZERO = timedelta(0)
-STDOFFSET = timedelta(seconds = -timezone)
-DSTOFFSET = timedelta(seconds = -altzone) if daylight else STDOFFSET
-DSTDIFF = DSTOFFSET - STDOFFSET
class LocalTimezone(tzinfo):
- def utcoffset(self, dt):
- return DSTOFFSET if self._isdst(dt) else STDOFFSET
+ ZERO = timedelta(0)
+ STDOFFSET = timedelta(seconds=-timezone)
+ DSTOFFSET = timedelta(seconds=-altzone) if daylight else STDOFFSET
+ DSTDIFF = DSTOFFSET - STDOFFSET
+ def utcoffset(self, dt):
+ return self.DSTOFFSET if self._isdst(dt) else self.STDOFFSET
def dst(self, dt):
- return DSTDIFF if self._isdst(dt) else ZERO
-
+ return self.DSTDIFF if self._isdst(dt) else self.ZERO
def tzname(self, dt):
return tzname[self._isdst(dt)]
-
def _isdst(self, dt):
stamp = mktime((dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second, dt.weekday(), 0, -1))
tt = localtime(stamp)
return tt.tm_isdst > 0
-
def localize(self, dt, is_dst=False):
"""Implemented for compatibility with pytz
"""
if dt.tzinfo is not None:
- raise ValueError, 'Not naive datetime (tzinfo is already set)'
+ raise ValueError('Not naive datetime (tzinfo is already set)')
return dt.replace(tzinfo=self)
diff --git a/itools/core/utils.py b/itools/core/utils.py
index 92b92d72c..cfa9c2709 100644
--- a/itools/core/utils.py
+++ b/itools/core/utils.py
@@ -50,7 +50,6 @@ def get_abspath(local_path, mname=None):
return mpath
-
def get_version(mname=None):
if mname is None:
mname = _getframe(1).f_globals.get('__name__')
@@ -62,7 +61,6 @@ def get_version(mname=None):
return None
-
def merge_dicts(d, *args, **kw):
"""Merge two or more dictionaries into a new dictionary object.
"""
@@ -73,7 +71,6 @@ def merge_dicts(d, *args, **kw):
return new_d
-
def get_sizeof(obj):
"""Return the size of an object and all objects refered by it.
"""
@@ -93,13 +90,12 @@ def get_sizeof(obj):
return size
-
def get_pipe(command, cwd=None):
"""Wrapper around 'subprocess.Popen'
"""
popen = Popen(command, stdout=PIPE, stderr=PIPE, cwd=cwd)
stdoutdata, stderrdata = popen.communicate()
if popen.returncode != 0:
- raise EnvironmentError, (popen.returncode, stderrdata)
+ raise EnvironmentError((popen.returncode, stderrdata))
return stdoutdata
diff --git a/itools/csv/__init__.py b/itools/csv/__init__.py
index 55ec5c177..8592b3818 100644
--- a/itools/csv/__init__.py
+++ b/itools/csv/__init__.py
@@ -17,11 +17,11 @@
# Import from itools
from itools.core import add_type
-from csv_ import CSVFile, Row
-from parser import parse
-from table import Table, Record, UniqueError
-from table import parse_table, fold_line, escape_data, is_multilingual
-from table import Property, property_to_str, deserialize_parameters
+from .csv_ import CSVFile, Row
+from .parser import parse
+from .table import Table, Record, UniqueError
+from .table import parse_table, fold_line, escape_data, is_multilingual
+from .table import Property, property_to_str, deserialize_parameters
__all__ = [
@@ -44,5 +44,4 @@
]
-
add_type('text/comma-separated-values', '.csv')
diff --git a/itools/csv/csv_.py b/itools/csv/csv_.py
old mode 100644
new mode 100755
index 1a7cc44d9..532653d7f
--- a/itools/csv/csv_.py
+++ b/itools/csv/csv_.py
@@ -21,8 +21,7 @@
# Import from itools
from itools.datatypes import String, Unicode
from itools.handlers import TextFile, guess_encoding, register_handler_class
-from parser import parse
-
+from .parser import parse
# TODO Drop the 'Row' class, use a list or a tuple instead.
@@ -37,7 +36,6 @@ def get_value(self, name):
return self[column]
-
def copy(self):
clone = self.__class__(self)
clone.number = self.number
@@ -45,7 +43,6 @@ def copy(self):
return clone
-
class CSVFile(TextFile):
class_mimetypes = ['text/comma-separated-values', 'text/x-comma-separated-values',
@@ -66,7 +63,6 @@ class CSVFile(TextFile):
has_header = False
skip_header = False
-
#########################################################################
# Load & Save
#########################################################################
@@ -75,11 +71,9 @@ def reset(self):
self.lines = []
self.n_lines = 0
-
def new(self):
pass
-
def _load_state_from_file(self, file):
# Guess encoding
data = file.read()
@@ -91,21 +85,22 @@ def _load_state_from_file(self, file):
# Header
if self.has_header:
- self.header = parser.next()
+ self.header = next(parser)
# Content
for line in parser:
self._add_row(line)
-
def to_str(self, encoding='UTF-8', separator=',', newline='\n'):
+
def escape(data):
+
return '"%s"' % data.replace('"', '""')
lines = []
# Header
if self.has_header:
- line = [ escape(Unicode.encode(x, encoding)) for x in self.header ]
+ line = [escape(Unicode.encode(x, encoding)) for x in self.header]
line = separator.join(line)
lines.append(line)
@@ -114,23 +109,25 @@ def escape(data):
schema = self.schema
columns = self.columns
if schema and columns:
- datatypes = [ (i, schema[x]) for i, x in enumerate(columns) ]
+ datatypes = [(i, schema[x]) for i, x in enumerate(columns)]
for row in self.get_rows():
line = []
for i, datatype in datatypes:
- try:
- data = datatype.encode(row[i], encoding=encoding)
- except TypeError:
- data = datatype.encode(row[i])
- line.append(escape(data))
+ if isinstance(row[i], str):
+ line.append(escape(row[i]))
+ else:
+ try:
+ data = datatype.encode(row[i], encoding=encoding)
+ except TypeError:
+ data = datatype.encode(row[i])
+ line.append(escape(data))
lines.append(separator.join(line))
else:
for row in self.get_rows():
- line = [ escape(x) for x in row ]
+ line = [escape(x) for x in row]
lines.append(separator.join(line))
return newline.join(lines)
-
#########################################################################
# API / Private
#########################################################################
@@ -147,20 +144,17 @@ def _add_row(self, row):
return row
-
def get_datatype(self, name):
if self.schema is None:
# Default
return String
return self.schema[name]
-
#########################################################################
# API / Public
#########################################################################
def get_nrows(self):
- return len([ x for x in self.lines if x is not None])
-
+ return len([x for x in self.lines if x is not None])
def get_row(self, number):
"""Return row at the given line number. Count begins at 0.
@@ -170,10 +164,9 @@ def get_row(self, number):
"""
row = self.lines[number]
if row is None:
- raise IndexError, 'list index out of range'
+ raise IndexError('list index out of range')
return row
-
def get_rows(self, numbers=None):
"""Return rows at the given list of line numbers. If no numbers
are given, then all rows are returned.
@@ -188,14 +181,12 @@ def get_rows(self, numbers=None):
for i in numbers:
yield self.get_row(i)
-
def add_row(self, row):
"""Append new row as an instance of row class.
"""
self.set_changed()
return self._add_row(row)
-
def update_row(self, index, **kw):
row = self.get_row(index)
self.set_changed()
@@ -210,7 +201,6 @@ def update_row(self, index, **kw):
column = columns.index(name)
row[column] = kw[name]
-
def del_row(self, number):
"""Delete row at the given line number. Count begins at 0.
"""
@@ -219,10 +209,9 @@ def del_row(self, number):
# Remove
row = self.lines[number]
if row is None:
- raise IndexError, 'list assignment index out of range'
+ raise IndexError('list assignment index out of range')
self.lines[number] = None
-
def del_rows(self, numbers):
"""Delete rows at the given line numbers. Count begins at 0.
"""
@@ -231,9 +220,8 @@ def del_rows(self, numbers):
for i in numbers:
self.del_row(i)
-
def get_unique_values(self, name):
- return set([ x.get_value(name) for x in self.get_rows() ])
+ return set([x.get_value(name) for x in self.get_rows()])
register_handler_class(CSVFile)
diff --git a/itools/csv/parser.py b/itools/csv/parser.py
old mode 100644
new mode 100755
index 4a757f942..8f2164df7
--- a/itools/csv/parser.py
+++ b/itools/csv/parser.py
@@ -30,7 +30,7 @@ def parse_line(reader, line, datatypes, encoding, n_columns):
if len(line) != n_columns:
msg = 'CSV syntax error: wrong number of columns at line %d: %s'
line_num = getattr(reader, 'line_num', None)
- raise ValueError, msg % (line_num, line)
+ raise ValueError(msg % (line_num, line))
# Decode values
decoded = []
@@ -44,7 +44,7 @@ def parse_line(reader, line, datatypes, encoding, n_columns):
# Next line
try:
- next_line = reader.next()
+ next_line = next(reader)
except StopIteration:
next_line = None
except Exception:
@@ -55,7 +55,6 @@ def parse_line(reader, line, datatypes, encoding, n_columns):
return decoded, next_line
-
def parse(data, columns=None, schema=None, guess=False, has_header=False,
encoding='UTF-8', **kw):
"""This method is a generator that returns one CSV row at a time. To
@@ -78,12 +77,12 @@ def parse(data, columns=None, schema=None, guess=False, has_header=False,
reader = read_csv(lines, **kw)
# 2. Find out the number of columns, if not specified
- line = reader.next()
+ line = next(reader)
n_columns = len(columns) if columns is not None else len(line)
# 3. The header
if has_header is True:
- datatypes = [ Unicode for x in range(n_columns) ]
+ datatypes = [Unicode for x in range(n_columns)]
datatypes = enumerate(datatypes)
datatypes = list(datatypes)
header, line = parse_line(reader, line, datatypes, encoding, n_columns)
@@ -91,9 +90,9 @@ def parse(data, columns=None, schema=None, guess=False, has_header=False,
# 4. The content
if schema is not None:
- datatypes = [ schema.get(c, String) for c in columns ]
+ datatypes = [schema.get(c, String) for c in columns]
else:
- datatypes = [ String for x in range(n_columns) ]
+ datatypes = [String for x in range(n_columns)]
datatypes = enumerate(datatypes)
datatypes = list(datatypes)
diff --git a/itools/csv/table.py b/itools/csv/table.py
old mode 100644
new mode 100755
index 06eae6a2c..205d8f07c
--- a/itools/csv/table.py
+++ b/itools/csv/table.py
@@ -26,9 +26,9 @@
# Import from itools
from itools.datatypes import DateTime, String, Unicode
-from itools.handlers import File
-from csv_ import CSVFile
-from parser import parse
+from itools.handlers import TextFile
+from .csv_ import CSVFile
+from .parser import parse
###########################################################################
@@ -51,7 +51,6 @@ def unescape_data(data, escape_table=escape_table):
return '\\'.join(out)
-
def escape_data(data, escape_table=escape_table):
"""Escape the data
"""
@@ -61,7 +60,6 @@ def escape_data(data, escape_table=escape_table):
return data
-
def unfold_lines(data):
"""Unfold the folded lines.
"""
@@ -82,7 +80,6 @@ def unfold_lines(data):
yield line
-
def fold_line(data):
"""Fold the unfolded line over 75 characters.
"""
@@ -107,7 +104,6 @@ def fold_line(data):
return res
-
# XXX The RFC only allows '-', we allow more because this is used by
# itools.database (metadata). This set matches checkid (itools.handlers)
allowed = frozenset(['-', '_', '.', '@'])
@@ -121,7 +117,7 @@ def read_name(line, allowed=allowed):
# Test first character of name
c = line[0]
if not c.isalnum() and c != '-':
- raise SyntaxError, 'unexpected character (%s)' % c
+ raise SyntaxError('unexpected character (%s)' % c)
# Test the rest
idx = 1
@@ -133,9 +129,9 @@ def read_name(line, allowed=allowed):
if c.isalnum() or c in allowed:
idx += 1
continue
- raise SyntaxError, "unexpected character '%s' (%s)" % (c, ord(c))
+ raise SyntaxError("unexpected character '%s' (%s)" % (c, ord(c)))
- raise SyntaxError, 'unexpected end of line (%s)' % line
+ raise SyntaxError('unexpected end of line (%s)' % line)
# Manage an icalendar content line value property [with parameters] :
@@ -178,7 +174,7 @@ def get_tokens(property):
if c.isalnum() or c in ('-'):
param_name, status = c, 2
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
# param-name begun
elif status == 2:
@@ -188,7 +184,7 @@ def get_tokens(property):
parameters[param_name] = []
status = 3
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
# param-name ended, param-value beginning
elif status == 3:
@@ -233,7 +229,7 @@ def get_tokens(property):
parameters[param_name].append(param_value)
status = 3
elif c == '"':
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
else:
param_value += c
@@ -251,17 +247,16 @@ def get_tokens(property):
elif c == ',':
status = 3
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
if status not in (7, 8):
- raise SyntaxError, 'unexpected property (%s)' % property
+ raise SyntaxError('unexpected property (%s)' % property)
# Unescape special characters (TODO Check the spec)
value = unescape_data(value)
return value, parameters
-
def parse_table(data):
"""This is the public interface of the module "itools.ical.parser", a
low-level parser of iCalendar files.
@@ -281,7 +276,6 @@ def parse_table(data):
yield name, value, parameters
-
###########################################################################
# Helper functions
###########################################################################
@@ -297,21 +291,20 @@ def deserialize_parameters(parameters, schema, default=String(multiple=True)):
for name in parameters:
datatype = schema.get(name, default)
if datatype is None:
- raise ValueError, 'parameter "{0}" not defined'.format(name)
+ raise ValueError('parameter "{0}" not defined'.format(name))
# Decode
value = parameters[name]
- value = [ decode_param_value(x, datatype) for x in value ]
+ value = [decode_param_value(x, datatype) for x in value]
# Multiple or single
if not datatype.multiple:
if len(value) > 1:
msg = 'parameter "%s" must be a singleton'
- raise ValueError, msg % name
+ raise ValueError(msg % name)
value = value[0]
# Update
parameters[name] = value
-
###########################################################################
# UniqueError
###########################################################################
@@ -323,11 +316,9 @@ def __init__(self, name, value):
self.name = name
self.value = value
-
def __str__(self):
- return (
- u'the "{field}" field must be unique, the "{value}" value is '
- u' already used.').format(field=self.name, value=self.value)
+ return ('the "{field}" field must be unique, the "{value}" value is '
+ 'already used.').format(field=self.name, value=self.value)
###########################################################################
@@ -347,30 +338,26 @@ def __init__(self, value, **kw):
self.value = value
self.parameters = kw or None
-
def clone(self):
# Copy the value and parameters
value = deepcopy(self.value)
parameters = {}
- for p_key, p_value in self.parameters.iteritems():
+ for p_key, p_value in self.parameters.items():
c_value = deepcopy(p_value)
parameters[p_key] = c_value
return Property(value, **parameters)
-
def get_parameter(self, name, default=None):
if self.parameters is None:
return default
return self.parameters.get(name, default)
-
def set_parameter(self, name, value):
if self.parameters is None:
self.parameters = {}
self.parameters[name] = value
-
def __eq__(self, other):
if type(other) is not Property:
return False
@@ -378,12 +365,10 @@ def __eq__(self, other):
return False
return self.parameters == other.parameters
-
def __ne__(self, other):
return not self.__eq__(other)
-
params_escape_table = (
('"', r'\"'),
('\r', r'\r'),
@@ -401,7 +386,7 @@ def encode_param_value(p_name, p_value, p_datatype):
# Standard case (ical behavior)
if '"' in p_value or '\n' in p_value:
error = 'the "%s" parameter contains a double quote'
- raise ValueError, error % p_name
+ raise ValueError(error % p_name)
if ';' in p_value or ':' in p_value or ',' in p_value:
return '"%s"' % p_value
return p_value
@@ -426,7 +411,7 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
# Parameters
if property.parameters:
p_names = property.parameters.keys()
- p_names.sort()
+ p_names = sorted(list(p_names))
else:
p_names = []
@@ -443,7 +428,7 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
# FIXME Use the encoding
if is_multiple(p_datatype):
p_value = [
- encode_param_value(p_name, x, p_datatype) for x in p_value ]
+ encode_param_value(p_name, x, p_datatype) for x in p_value]
p_value = ','.join(p_value)
else:
p_value = encode_param_value(p_name, p_value, p_datatype)
@@ -456,8 +441,8 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
else:
value = datatype.encode(property.value)
if type(value) is not str:
- raise ValueError, 'property "{0}" is not str but {1}'.format(
- name, type(value))
+ raise ValueError('property "{0}" is not str but {1}'.format(
+ name, type(value)))
value = escape_data(value)
# Ok
@@ -468,39 +453,35 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
def property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
try:
return _property_to_str(name, property, datatype, p_schema, encoding)
- except StandardError:
+ except Exception:
err = 'failed to serialize "%s" property, probably a bad value'
- raise ValueError, err % name
-
+ raise ValueError(err % name)
class Record(dict):
__slots__ = ['id', 'record_properties']
-
- def __init__(self, id, record_properties):
- self.id = id
- self.record_properties = record_properties
-
+ def __init__(self, _id, record_properties):
+ super().__init__()
+ self.id = _id
+ self.record_properties = record_properties
def __getattr__(self, name):
if name == '__number__':
return self.id
if name not in self:
- raise AttributeError, "'%s' object has no attribute '%s'" % (
- self.__class__.__name__, name)
+ raise AttributeError("'%s' object has no attribute '%s'" % (
+ self.__class__.__name__, name))
property = self[name]
if type(property) is list:
- return [ x.value for x in property ]
+ return [x.value for x in property]
return property.value
-
def get_property(self, name):
return self.get(name)
-
# For indexing purposes
def get_value(self, name):
property = self.get(name)
@@ -508,12 +489,11 @@ def get_value(self, name):
return None
if type(property) is list:
- return [ x.value for x in property ]
+ return [x.value for x in property]
return property.value
-
-class Table(File):
+class Table(TextFile):
record_class = Record
@@ -526,7 +506,6 @@ class Table(File):
record_parameters = {
'language': String(multiple=False)}
-
def get_datatype(self, name):
# Table schema
if name == 'ts':
@@ -535,7 +514,6 @@ def get_datatype(self, name):
return self.schema[name]
return String(multiple=True)
-
def get_record_datatype(self, name):
# Record schema
if name == 'ts':
@@ -545,7 +523,6 @@ def get_record_datatype(self, name):
# FIXME Probably we should raise an exception here
return String(multiple=True)
-
def properties_to_dict(self, properties, record, first=False):
"""Add the given "properties" as Property objects or Property objects
list to the given dictionnary "record".
@@ -566,23 +543,22 @@ def properties_to_dict(self, properties, record, first=False):
# Transform values to properties
if is_multilingual(datatype):
if type(value) is not list:
- value = [ value ]
+ value = [value]
record.setdefault(name, [])
for p in value:
language = p.parameters['language']
record[name] = [
x for x in record[name]
- if x.parameters['language'] != language ]
+ if x.parameters['language'] != language]
record[name].append(p)
elif datatype.multiple:
if type(value) is list:
- record[name] = [ to_property(x) for x in value ]
+ record[name] = [to_property(x) for x in value]
else:
record[name] = [to_property(value)]
else:
record[name] = to_property(value)
-
#######################################################################
# Handlers
#######################################################################
@@ -591,14 +567,12 @@ def reset(self):
self.records = []
self.changed_properties = False
-
def new(self):
# Add the properties record
properties = self.record_class(-1, self.record_properties)
properties['ts'] = Property(datetime.now())
self.properties = properties
-
def _load_state_from_file(self, file):
# Load the records
records = self.records
@@ -644,15 +618,14 @@ def _load_state_from_file(self, file):
record.setdefault(name, []).append(property)
elif name in record:
msg = "record %s: property '%s' can occur only once"
- raise ValueError, msg % (uid, name)
+ raise ValueError(msg % (uid, name))
else:
record[name] = property
-
def _record_to_str(self, id, record):
lines = ['id:%d/0\n' % id]
names = record.keys()
- names.sort()
+ names = sorted(list(names))
# Table or record schema
if id == -1:
get_datatype = self.get_datatype
@@ -677,7 +650,6 @@ def _record_to_str(self, id, record):
lines.append('\n')
return ''.join(lines)
-
def to_str(self):
lines = []
# Properties record
@@ -695,17 +667,15 @@ def to_str(self):
return ''.join(lines)
-
#######################################################################
# API / Public
#######################################################################
- def get_record(self, id):
+ def get_record(self, _id):
try:
- return self.records[id]
+ return self.records[_id]
except IndexError:
return None
-
def add_record(self, kw):
# Check for duplicate
for name in kw:
@@ -714,8 +684,8 @@ def add_record(self, kw):
if self.search(name, kw[name]):
raise UniqueError(name, kw[name])
# Make new record
- id = len(self.records)
- record = self.record_class(id, self.record_properties)
+ _id = len(self.records)
+ record = self.record_class(_id, self.record_properties)
self.properties_to_dict(kw, record)
record['ts'] = Property(datetime.now())
# Change
@@ -724,25 +694,23 @@ def add_record(self, kw):
# Back
return record
-
- def update_record(self, id, **kw):
- record = self.records[id]
+ def update_record(self, _id, **kw):
+ record = self.records[_id]
if record is None:
msg = 'cannot modify record "%s" because it has been deleted'
- raise LookupError, msg % id
+ raise LookupError(msg % _id)
# Check for duplicate
for name in kw:
datatype = self.get_record_datatype(name)
if getattr(datatype, 'unique', False) is True:
search = self.search(name, kw[name])
- if search and (search[0] != self.records[id]):
+ if search and (search[0] != self.records[_id]):
raise UniqueError(name, kw[name])
# Update record
self.set_changed()
self.properties_to_dict(kw, record)
record['ts'] = Property(datetime.now())
-
def update_properties(self, **kw):
record = self.properties
if record is None:
@@ -757,16 +725,14 @@ def update_properties(self, **kw):
self.set_changed()
self.changed_properties = True
-
- def del_record(self, id):
- record = self.records[id]
+ def del_record(self, _id):
+ record = self.records[_id]
if record is None:
msg = 'cannot delete record "%s" because it was deleted before'
- raise LookupError, msg % id
+ raise LookupError(msg % _id)
# Change
self.set_changed()
- self.records[id] = None
-
+ self.records[_id] = None
def get_record_ids(self):
i = 0
@@ -775,16 +741,13 @@ def get_record_ids(self):
yield i
i += 1
-
def get_n_records(self):
ids = self.get_record_ids()
ids = list(ids)
return len(ids)
-
def get_records(self):
- return ( x for x in self.records if x )
-
+ return (x for x in self.records if x)
def get_record_value(self, record, name, language=None):
"""This is the preferred method for accessing record values. It
@@ -808,8 +771,8 @@ def get_record_value(self, record, name, language=None):
return datatype.get_default()
# Language negotiation ('select_language' is a built-in)
if language is None:
- languages = [ x.parameters['language'] for x in property
- if not datatype.is_empty(x.value) ]
+ languages = [x.parameters['language'] for x in property
+ if not datatype.is_empty(x.value)]
language = select_language(languages)
if language is None and languages:
# Pick up one at random (FIXME)
@@ -834,19 +797,17 @@ def get_record_value(self, record, name, language=None):
return []
return default
# Hit
- return [ x.value for x in property ]
+ return [x.value for x in property]
# Simple properties
if property is None:
return datatype.get_default()
return property.value
-
def get_property(self, name):
record = self.properties
return record.get_value(name)
-
def get_property_value(self, name):
"""Return the value if name is in record
else if name is define in the schema
@@ -865,11 +826,9 @@ def get_property_value(self, name):
else:
return getattr(datatype, 'default')
-
def search(self, key, value):
get = self.get_record_value
- return [ x for x in self.records if x and get(x, key) == value ]
-
+ return [x for x in self.records if x and get(x, key) == value]
def update_from_csv(self, data, columns, skip_header=False):
"""Update the table by adding record from data
@@ -888,7 +847,6 @@ def update_from_csv(self, data, columns, skip_header=False):
record[key] = line[index]
self.add_record(record)
-
def to_csv(self, columns, separator=None, language=None):
"""Export the table to CSV handler.
As table columns are unordered, the order comes from the "columns"
@@ -912,7 +870,7 @@ def to_csv(self, columns, separator=None, language=None):
# TODO represent multiple values
message = ("multiple values are not supported, "
"use a separator")
- raise NotImplementedError, message
+ raise NotImplementedError(message)
else:
data = datatype.encode(value)
line.append(data)
diff --git a/itools/database/__init__.py b/itools/database/__init__.py
index 929522828..a30117b34 100644
--- a/itools/database/__init__.py
+++ b/itools/database/__init__.py
@@ -17,16 +17,16 @@
# along with this program. If not, see .
# Import from itools
-from fields import Field, get_field_and_datatype
-from queries import AllQuery, NotQuery, StartQuery, TextQuery
-from queries import RangeQuery, PhraseQuery, AndQuery, OrQuery, pprint_query
-from magic_ import magic_from_buffer, magic_from_file
-from metadata import Metadata
-from metadata_parser import MetadataProperty
-from registry import get_register_fields, register_field
-from resources import Resource
-from ro import RODatabase, ReadonlyError
-from rw import RWDatabase, make_database
+from .fields import Field, get_field_and_datatype
+from .queries import AllQuery, NotQuery, StartQuery, TextQuery
+from .queries import RangeQuery, PhraseQuery, AndQuery, OrQuery, pprint_query
+from .magic_ import magic_from_buffer, magic_from_file
+from .metadata import Metadata
+from .metadata_parser import MetadataProperty
+from .registry import get_register_fields, register_field
+from .resources import Resource
+from .ro import RODatabase, ReadonlyError
+from .rw import RWDatabase, make_database
__all__ = [
diff --git a/itools/database/backends/__init__.py b/itools/database/backends/__init__.py
index 356eff554..4c0da5bbe 100644
--- a/itools/database/backends/__init__.py
+++ b/itools/database/backends/__init__.py
@@ -17,6 +17,6 @@
# along with this program. If not, see .
# Import from itools
-from git import GitBackend
-from lfs import LFSBackend
-from registry import register_backend, backends_registry
+from .git import GitBackend
+from .lfs import LFSBackend
+from .registry import register_backend, backends_registry
diff --git a/itools/database/backends/catalog.py b/itools/database/backends/catalog.py
index 6c420faf0..c6c377d23 100644
--- a/itools/database/backends/catalog.py
+++ b/itools/database/backends/catalog.py
@@ -19,14 +19,13 @@
# Import from the standard library
import os
-from copy import deepcopy
from decimal import Decimal as decimal
from datetime import datetime
from marshal import dumps, loads
from hashlib import sha1
# Import from xapian
-from xapian import Database, WritableDatabase, DB_CREATE, DB_OPEN, DB_BACKEND_CHERT
+from xapian import Database, WritableDatabase, DB_CREATE, DB_OPEN, DB_BACKEND_GLASS
from xapian import Document, Query, QueryParser, Enquire
from xapian import sortable_serialise, sortable_unserialise, TermGenerator
@@ -35,18 +34,19 @@
from itools.datatypes import Decimal, Integer, Unicode, String
from itools.fs import lfs
from itools.i18n import is_punctuation
-from itools.log import Logger, log_warning, log_info, register_logger
+from logging import getLogger
from itools.database.queries import AllQuery, _AndQuery, NotQuery, _OrQuery, PhraseQuery
from itools.database.queries import RangeQuery, StartQuery, TextQuery, _MultipleQuery
+log = getLogger("itools.database")
try:
from xapian import MultiValueSorter
+
XAPIAN_VERSION = '1.2'
-except:
+except Exception:
from xapian import MultiValueKeyMaker
- XAPIAN_VERSION = '1.4'
-
+ XAPIAN_VERSION = '1.4'
# Constants
OP_AND = Query.OP_AND
@@ -59,46 +59,58 @@
TQ_FLAGS = (QueryParser.FLAG_LOVEHATE +
QueryParser.FLAG_PHRASE +
QueryParser.FLAG_WILDCARD)
-TRANSLATE_MAP = { ord(u'À'): ord(u'A'),
- ord(u'Â'): ord(u'A'),
- ord(u'â'): ord(u'a'),
- ord(u'à'): ord(u'a'),
- ord(u'Ç'): ord(u'C'),
- ord(u'ç'): ord(u'c'),
- ord(u'É'): ord(u'E'),
- ord(u'Ê'): ord(u'E'),
- ord(u'é'): ord(u'e'),
- ord(u'ê'): ord(u'e'),
- ord(u'è'): ord(u'e'),
- ord(u'ë'): ord(u'e'),
- ord(u'Î'): ord(u'I'),
- ord(u'î'): ord(u'i'),
- ord(u'ï'): ord(u'i'),
- ord(u'ô'): ord(u'o'),
- ord(u'û'): ord(u'u'),
- ord(u'ù'): ord(u'u'),
- ord(u'ü'): ord(u'u'),
- ord(u"'"): ord(u' ') }
+TRANSLATE_MAP = {ord('À'): ord('A'),
+ ord('Â'): ord('A'),
+ ord('â'): ord('a'),
+ ord('à'): ord('a'),
+ ord('Ç'): ord('C'),
+ ord('ç'): ord('c'),
+ ord('É'): ord('E'),
+ ord('Ê'): ord('E'),
+ ord('é'): ord('e'),
+ ord('ê'): ord('e'),
+ ord('è'): ord('e'),
+ ord('ë'): ord('e'),
+ ord('Î'): ord('I'),
+ ord('î'): ord('i'),
+ ord('ï'): ord('i'),
+ ord('ô'): ord('o'),
+ ord('û'): ord('u'),
+ ord('ù'): ord('u'),
+ ord('ü'): ord('u'),
+ ord("'"): ord(' ')}
+MSG_NOT_INDEXED = 'the "{name}" field is not indexed'
+
+
+def bytes_to_str(data):
+ for encoding in ["utf-8", "windows-1252", "latin-1"]:
+ try:
+ if isinstance(data, bytes):
+ return data.decode(encoding)
+ else:
+ return data
+ except:
+ pass
+ raise Exception(f"Type DATA {type(data)} value {data}")
-MSG_NOT_INDEXED = 'the "{name}" field is not indexed'
def warn_not_indexed(name):
- log_warning(MSG_NOT_INDEXED.format(name=name), 'itools.database')
+ log.warning(MSG_NOT_INDEXED.format(name=name))
+
MSG_NOT_STORED = 'the "{name}" field is not stored'
+
+
def warn_not_stored(name):
- log_warning(MSG_NOT_STORED.format(name=name), 'itools.database')
+ log.warning(MSG_NOT_STORED.format(name=name))
-MSG_NOT_INDEXED_NOR_STORED = 'the "{name}" field is not indexed nor stored'
-def warn_not_indexed_nor_stored(name):
- log_warning(MSG_NOT_INDEXED_NOR_STORED.format(name=name), 'itools.database')
+MSG_NOT_INDEXED_NOR_STORED = 'the "{name}" field is not indexed nor stored'
-class CatalogLogger(Logger):
- def format(self, domain, level, message):
- return message + '\n'
+def warn_not_indexed_nor_stored(name):
+ log.warning(MSG_NOT_INDEXED_NOR_STORED.format(name=name))
class Doc(object):
@@ -108,16 +120,15 @@ def __init__(self, xdoc, fields, metadata):
self._fields = fields
self._metadata = metadata
-
def __getattr__(self, name):
# 1. Get the raw value
info = self._metadata.get(name)
if info is None:
- raise AttributeError, MSG_NOT_INDEXED_NOR_STORED.format(name=name)
+ raise AttributeError(MSG_NOT_INDEXED_NOR_STORED.format(name=name))
stored = info.get('value')
if stored is None:
- raise AttributeError, MSG_NOT_STORED.format(name=name)
+ raise AttributeError(MSG_NOT_STORED.format(name=name))
raw_value = self._xdoc.get_value(stored)
# 2. Decode
@@ -155,7 +166,6 @@ def __getattr__(self, name):
setattr(self, name, value)
return value
-
def get_value(self, name, language=None):
# Check if stored
info = self._metadata.get(name)
@@ -198,21 +208,18 @@ def get_value(self, name, language=None):
return field_cls.get_default()
-
class SearchResults(object):
def __init__(self, catalog, xquery):
self._catalog = catalog
self._xquery = xquery
-
@lazy
def _enquire(self):
enquire = Enquire(self._catalog._db)
enquire.set_query(self._xquery)
return enquire
-
@lazy
def _max(self):
enquire = self._enquire
@@ -220,18 +227,15 @@ def _max(self):
doccount = db.get_doccount()
return enquire.get_mset(0, doccount).size()
-
def __len__(self):
"""Returns the number of documents found."""
return self._max
-
def search(self, query=None, **kw):
xquery = _get_xquery(self._catalog, query, **kw)
query = Query(Query.OP_AND, [self._xquery, xquery])
return self.__class__(self._catalog, query)
-
def get_documents(self, sort_by=None, reverse=False, start=0, size=0):
"""Returns the documents for the search, sorted by weight.
@@ -297,8 +301,8 @@ def get_documents(self, sort_by=None, reverse=False, start=0, size=0):
# Construction of the results
fields = self._catalog._fields
- results = [ Doc(x.document, fields, metadata)
- for x in enquire.get_mset(start, size) ]
+ results = [Doc(x.document, fields, metadata)
+ for x in enquire.get_mset(start, size)]
# sort_by=None/reverse=True
if sort_by is None and reverse:
@@ -307,11 +311,8 @@ def get_documents(self, sort_by=None, reverse=False, start=0, size=0):
return results
-
class Catalog(object):
-
nb_changes = 0
- logger = None
_db = None
read_only = False
@@ -332,7 +333,7 @@ def __init__(self, ref, fields, read_only=False, asynchronous_mode=True):
self._asynchronous = asynchronous_mode
self._fields = fields
# FIXME: There's a bug in xapian:
- # Wa cannot get stored values if DB not flushed
+ # We cannot get stored values if DB not flushed
self.commit_each_transaction = True
# Asynchronous mode
if not read_only and asynchronous_mode:
@@ -346,37 +347,27 @@ def __init__(self, ref, fields, read_only=False, asynchronous_mode=True):
self._load_all_internal()
if not read_only:
self._init_all_metadata()
- # Catalog log
- if path:
- catalog_log = '{}/catalog.log'.format(path)
- self.logger = CatalogLogger(catalog_log)
- register_logger(self.logger, 'itools.catalog')
-
-
- def _init_all_metadata(self):
+ def _init_all_metadata(self, has_changes=False):
"""Init new metadata (to avoid 'field is not indexed' warning)
"""
- has_changes = False
metadata = self._metadata
for name, field_cls in self._fields.items():
if name not in metadata:
- print('[Catalog] New field registered: {0}'.format(name))
+ log.debug("[Catalog] New field registered: {0}".format(name))
has_changes = True
metadata[name] = self._get_info(field_cls, name)
else:
# If the field was in the catalog but is newly stored
- if (not metadata[name].has_key('value') and
- getattr(field_cls, 'stored', False)):
- print('[Catalog] Indexed field is now stored: {0}'.format(name))
+ if 'value' not in metadata[name] and getattr(field_cls, 'stored', False):
+ log.debug("[Catalog] Indexed field is now stored: {0}".format(name))
has_changes = True
metadata[name] = merge_dicts(
metadata[name],
self._get_info_stored())
# If the field was stored in the catalog but is newly indexed
- if (not metadata[name].has_key('prefix') and
- getattr(field_cls, 'indexed', False)):
- print('[Catalog] Stored field is now indexed: {0}'.format(name))
+ if 'prefix' not in metadata[name] and getattr(field_cls, 'indexed', False):
+ log.debug("[Catalog] Stored field is now indexed: {0}".format(name))
has_changes = True
metadata[name] = merge_dicts(
metadata[name],
@@ -386,7 +377,6 @@ def _init_all_metadata(self):
self._db.commit_transaction()
self._db.begin_transaction(self.commit_each_transaction)
-
#######################################################################
# API / Public / Transactions
#######################################################################
@@ -394,41 +384,35 @@ def save_changes(self):
"""Save the last changes to disk.
"""
if not self._asynchronous:
- raise ValueError, "The transactions are synchronous"
+ raise ValueError("The transactions are synchronous")
db = self._db
db.commit_transaction()
- db.commit()
- # FIXME: There's a bug in xapian:
- # Wa cannot get stored values if DB not flushed
- #if self.nb_changes > 200:
- # # XXX Not working since cancel_transaction()
- # # cancel all transactions not commited to disk
- # # We have to use new strategy to abort transaction
- # db.commit()
- # if self.logger:
- # self.logger.clear()
- # self.nb_changes = 0
+ if self.commit_each_transaction:
+ db.commit()
+ else:
+ if self.nb_changes > 200:
+ db.commit()
+ self.nb_changes = 0
db.begin_transaction(self.commit_each_transaction)
-
def abort_changes(self):
+
"""Abort the last changes made in memory.
"""
if not self._asynchronous:
- raise ValueError, "The transactions are synchronous"
+ raise ValueError("The transactions are synchronous")
db = self._db
if self.commit_each_transaction:
db.cancel_transaction()
db.begin_transaction(self.commit_each_transaction)
else:
- raise NotImplementedError
+ db.cancel_transaction()
+ db.begin_transaction(self.commit_each_transaction)
self._load_all_internal()
-
def close(self):
if self._db is None:
- msg = 'Catalog is already closed'
- print(msg)
+ log.info("Catalog is already closed")
return
if self.read_only:
self._db.close()
@@ -437,22 +421,18 @@ def close(self):
if self.commit_each_transaction:
try:
self._db.cancel_transaction()
- except:
- print('Warning: cannot cancel xapian transaction')
+ except Exception:
+ log.info("Warning: cannot cancel xapian transaction", exc_info=True)
self._db.close()
self._db = None
else:
self._db.close()
self._db = None
else:
- self.abort_changes()
self._db.commit_transaction()
- self._db.flush()
+ self._db.commit()
self._db.close()
self._db = None
- if self.logger:
- self.logger.clear()
-
#######################################################################
# API / Public / (Un)Index
@@ -461,9 +441,7 @@ def index_document(self, document):
self.nb_changes += 1
abspath, term, xdoc = self.get_xdoc_from_document(document)
self._db.replace_document(term, xdoc)
- if self.logger:
- log_info(abspath, domain='itools.catalog')
-
+ log.debug("Indexed : {}".format(abspath))
def unindex_document(self, abspath):
"""Remove the document that has value stored in its abspath.
@@ -471,10 +449,10 @@ def unindex_document(self, abspath):
"""
self.nb_changes += 1
data = _reduce_size(_encode(self._fields['abspath'], abspath))
+ if type(data) is bytes:
+ data = data.decode("utf-8")
self._db.delete_document('Q' + data)
- if self.logger:
- log_info(abspath, domain='itools.catalog')
-
+ log.debug("Unindexed : {}".format(abspath))
def get_xdoc_from_document(self, doc_values):
"""Return (abspath, term, xdoc) from the document (resource or values as dict)
@@ -489,7 +467,7 @@ def get_xdoc_from_document(self, doc_values):
# Make the xapian document
metadata_modified = False
xdoc = Document()
- for name, value in doc_values.iteritems():
+ for name, value in doc_values.items():
if name not in fields:
warn_not_indexed_nor_stored(name)
field_cls = fields[name]
@@ -510,12 +488,13 @@ def get_xdoc_from_document(self, doc_values):
# the problem is that "_encode != _index"
if name == 'abspath':
key_value = _reduce_size(_encode(field_cls, value))
+ key_value = bytes_to_str(key_value)
term = 'Q' + key_value
xdoc.add_term(term)
# A multilingual value?
if isinstance(value, dict):
- for language, lang_value in value.iteritems():
+ for language, lang_value in value.items():
lang_name = name + '_' + language
# New field ?
@@ -571,8 +550,7 @@ def get_unique_values(self, name):
# Ok
prefix = metadata[name]['prefix']
prefix_len = len(prefix)
- return set([ t.term[prefix_len:] for t in self._db.allterms(prefix) ])
-
+ return set([t.term[prefix_len:] for t in self._db.allterms(prefix)])
#######################################################################
# API / Private
@@ -583,8 +561,8 @@ def _get_info(self, field_cls, name):
if not (issubclass(field_cls, String) and
field_cls.stored and
field_cls.indexed):
- raise ValueError, ('the abspath field must be declared as '
- 'String(stored=True, indexed=True)')
+ raise ValueError(('the abspath field must be declared as '
+ 'String(stored=True, indexed=True)'))
# Stored ?
info = {}
if getattr(field_cls, 'stored', False):
@@ -595,20 +573,16 @@ def _get_info(self, field_cls, name):
# Ok
return info
-
def _get_info_stored(self):
value = self._value_nb
self._value_nb += 1
return {'value': value}
-
def _get_info_indexed(self):
prefix = _get_prefix(self._prefix_nb)
self._prefix_nb += 1
return {'prefix': prefix}
-
-
def _load_all_internal(self):
"""Load the metadata from the database
"""
@@ -616,17 +590,24 @@ def _load_all_internal(self):
self._prefix_nb = 0
metadata = self._db.get_metadata('metadata')
- if metadata == '':
+
+ if metadata == b'':
self._metadata = {}
else:
- self._metadata = loads(metadata)
- for name, info in self._metadata.iteritems():
+ try:
+ self._metadata = loads(metadata)
+ except ValueError:
+ # Reload metadata if incompatibility between Python 2 and Python 3
+ self._init_all_metadata(has_changes=True)
+ metadata = self._db.get_metadata('metadata')
+ self._metadata = loads(metadata)
+
+ for name, info in self._metadata.items():
if 'value' in info:
self._value_nb += 1
if 'prefix' in info:
self._prefix_nb += 1
-
def _query2xquery(self, query):
"""take a "itools" query and return a "xapian" query
"""
@@ -642,7 +623,7 @@ def _query2xquery(self, query):
if query_class is PhraseQuery:
name = query.name
if type(name) is not str:
- raise TypeError, "unexpected '%s'" % type(name)
+ raise TypeError("unexpected '%s'" % type(name))
# If there is a problem => an empty result
if name not in metadata:
warn_not_indexed(name)
@@ -651,7 +632,7 @@ def _query2xquery(self, query):
try:
prefix = info['prefix']
except KeyError:
- raise ValueError, 'the field "%s" must be indexed' % name
+ raise ValueError('the field "%s" must be indexed' % name)
field_cls = _get_field_cls(name, fields, info)
return _make_PhraseQuery(field_cls, query.value, prefix)
@@ -659,7 +640,7 @@ def _query2xquery(self, query):
if query_class is RangeQuery:
name = query.name
if type(name) is not str:
- raise TypeError, "unexpected '%s'" % type(name)
+ raise TypeError("unexpected '%s'" % type(name))
# If there is a problem => an empty result
if name not in metadata:
warn_not_indexed(name)
@@ -668,11 +649,11 @@ def _query2xquery(self, query):
info = metadata[name]
value = info.get('value')
if value is None:
- raise AttributeError, MSG_NOT_STORED.format(name=name)
+ raise AttributeError(MSG_NOT_STORED.format(name=name))
field_cls = _get_field_cls(name, fields, info)
if field_cls.multiple:
error = 'range-query not supported on multiple fields'
- raise ValueError, error
+ raise ValueError(error)
left = query.left
if left is not None:
@@ -701,7 +682,7 @@ def _query2xquery(self, query):
if query_class is StartQuery:
name = query.name
if type(name) is not str:
- raise TypeError, "unexpected '%s'" % type(name)
+ raise TypeError("unexpected '%s'" % type(name))
# If there is a problem => an empty result
if name not in metadata:
warn_not_indexed(name)
@@ -710,7 +691,7 @@ def _query2xquery(self, query):
info = metadata[name]
value_nb = info.get('value')
if value_nb is None:
- raise AttributeError, MSG_NOT_STORED.format(name=name)
+ raise AttributeError(MSG_NOT_STORED.format(name=name))
field_cls = _get_field_cls(name, fields, info)
value = query.value
@@ -750,7 +731,7 @@ def _query2xquery(self, query):
if query_class is TextQuery:
name = query.name
if type(name) is not str:
- raise TypeError, "unexpected %s for 'name'" % type(name)
+ raise TypeError("unexpected %s for 'name'" % type(name))
# If there is a problem => an empty result
if name not in metadata:
warn_not_indexed(name)
@@ -761,12 +742,12 @@ def _query2xquery(self, query):
try:
prefix = info['prefix']
except KeyError:
- raise ValueError, 'the field "%s" must be indexed' % name
+ raise ValueError('the field "%s" must be indexed' % name)
# Remove accents from the value
value = query.value
- if type(value) is not unicode:
- raise TypeError, "unexpected %s for 'value'" % type(value)
+ if type(value) is not str:
+ raise TypeError("unexpected %s for 'value'" % type(value))
value = value.translate(TRANSLATE_MAP)
qp = QueryParser()
@@ -780,18 +761,17 @@ def _query2xquery(self, query):
# And
if query_class is _AndQuery:
- return Query(OP_AND, [ i2x(q) for q in query.atoms ])
+ return Query(OP_AND, [i2x(q) for q in query.atoms])
# Or
if query_class is _OrQuery:
- return Query(OP_OR, [ i2x(q) for q in query.atoms ])
+ return Query(OP_OR, [i2x(q) for q in query.atoms])
# Not
if query_class is NotQuery:
return Query(OP_AND_NOT, Query(''), i2x(query.query))
-
def make_catalog(uri, fields):
"""Creates a new and empty catalog in the given uri.
@@ -804,17 +784,12 @@ def make_catalog(uri, fields):
'name': Unicode(indexed=True), ...}
"""
path = lfs.get_absolute_path(uri)
- db = WritableDatabase(path, DB_BACKEND_CHERT)
- # FIXME GLASS backend seems to be buggy
- # db = WritableDatabase(path, DB_CREATE)
+ db = WritableDatabase(path, DB_BACKEND_GLASS)
return Catalog(db, fields)
-
#############
# Private API
-
-
def _get_prefix(number):
"""By convention:
Q is used for the unique Id of a document
@@ -823,9 +798,8 @@ def _get_prefix(number):
"""
magic_letters = 'ABCDEFGHIJKLMNOPRSTUVWY'
size = len(magic_letters)
- result = 'X'*(number/size)
- return result+magic_letters[number%size]
-
+ result = 'X' * (number // size)
+ return result + magic_letters[number % size]
def _decode_simple_value(field_cls, data):
@@ -840,7 +814,6 @@ def _decode_simple_value(field_cls, data):
return field_cls.decode(data)
-
def _decode(field_cls, data):
# Singleton
if not field_cls.multiple:
@@ -851,8 +824,7 @@ def _decode(field_cls, data):
value = loads(data)
except (ValueError, MemoryError):
return _decode_simple_value(field_cls, data)
- return [ _decode_simple_value(field_cls, a_value) for a_value in value ]
-
+ return [_decode_simple_value(field_cls, a_value) for a_value in value]
# We must overload the normal behaviour (range + optimization)
@@ -872,7 +844,6 @@ def _encode_simple_value(field_cls, value):
return field_cls.encode(value)
-
def _encode(field_cls, value):
"""Used to encode values in stored fields.
"""
@@ -881,30 +852,29 @@ def _encode(field_cls, value):
return _encode_simple_value(field_cls, value)
# Case 2: Multiple
- value = [ _encode_simple_value(field_cls, a_value) for a_value in value ]
+ value = [_encode_simple_value(field_cls, a_value) for a_value in value]
return dumps(value)
-
def _get_field_cls(name, fields, info):
return fields[name] if (name in fields) else fields[info['from']]
-
def _reduce_size(data):
# 'data' must be a byte string
# If the data is too long, we replace it by its sha1
# FIXME Visibly a bug in xapian counts twice the \x00 character
# http://bugs.hforge.org/show_bug.cgi?id=940
- if len(data) + data.count("\x00") > 240:
+ if isinstance(data, str):
+ data = data.encode("utf-8")
+ if len(data) + data.count(b"\x00") > 240:
return sha1(data).hexdigest()
# All OK, we simply return the data
return data
-
def _index_cjk(xdoc, value, prefix, termpos):
"""
Returns the next word and its position in the data. The analysis
@@ -918,21 +888,21 @@ def _index_cjk(xdoc, value, prefix, termpos):
2 -> 0 [stop word]
"""
state = 0
- previous_cjk = u''
+ previous_cjk = ''
for c in value:
if is_punctuation(c):
# Stop word
- if previous_cjk and state == 1: # CJK not yielded yet
+ if previous_cjk and state == 1: # CJK not yielded yet
xdoc.add_posting(prefix + previous_cjk, termpos)
termpos += 1
# reset state
- previous_cjk = u''
+ previous_cjk = ''
state = 0
else:
c = c.lower()
if previous_cjk:
- xdoc.add_posting(prefix + (u'%s%s' % (previous_cjk, c)),
+ xdoc.add_posting(prefix + ('%s%s' % (previous_cjk, c)),
termpos)
termpos += 1
state = 2
@@ -947,13 +917,14 @@ def _index_cjk(xdoc, value, prefix, termpos):
return termpos + 1
-
def _index_unicode(xdoc, value, prefix, language, termpos,
TRANSLATE_MAP=TRANSLATE_MAP):
+
+ value = bytes_to_str(value)
# Check type
- if type(value) is not unicode:
+ if type(value) is not str:
msg = 'The value "%s", field "%s", is not a unicode'
- raise TypeError, msg % (value, prefix)
+ raise TypeError(msg % (value, prefix))
# Case 1: Japanese or Chinese
if language in ['ja', 'zh']:
@@ -967,13 +938,12 @@ def _index_unicode(xdoc, value, prefix, language, termpos,
value = value.translate(TRANSLATE_MAP)
# XXX With the stemmer, the words are saved twice:
# with prefix and with Zprefix
-# tg.set_stemmer(stemmer)
+ # tg.set_stemmer(stemmer)
tg.index_text(value, 1, prefix)
return tg.get_termpos() + 1
-
def _index(xdoc, field_cls, value, prefix, language):
"""To index a field it must be split in a sequence of words and
positions:
@@ -982,11 +952,12 @@ def _index(xdoc, field_cls, value, prefix, language):
Where will be a value.
"""
+ value = bytes_to_str(value)
is_multiple = (field_cls.multiple
and isinstance(value, (tuple, list, set, frozenset)))
# Case 1: Unicode (a complex split)
- if issubclass(field_cls, Unicode):
+ if issubclass(field_cls, Unicode) and value is not None:
if is_multiple:
termpos = 1
for x in value:
@@ -998,15 +969,16 @@ def _index(xdoc, field_cls, value, prefix, language):
for position, data in enumerate(value):
data = _encode_simple_value(field_cls, data)
data = _reduce_size(data)
+ data = bytes_to_str(data)
xdoc.add_posting(prefix + data, position + 1)
# Case 3: singleton
else:
data = _encode_simple_value(field_cls, value)
data = _reduce_size(data)
+ data = bytes_to_str(data)
xdoc.add_posting(prefix + data, 1)
-
def _make_PhraseQuery(field_cls, value, prefix):
# Get the words
# XXX It's too complex (slow), we must use xapian
@@ -1020,13 +992,12 @@ def _make_PhraseQuery(field_cls, value, prefix):
for termpos in term_list_item.positer:
words.append((termpos, term))
words.sort()
- words = [ word[1] for word in words ]
+ words = [word[1] for word in words]
# Make the query
return Query(OP_PHRASE, words)
-
def _get_xquery(catalog, query=None, **kw):
# Case 1: a query is given
if query is not None:
@@ -1040,7 +1011,7 @@ def _get_xquery(catalog, query=None, **kw):
metadata = catalog._metadata
fields = catalog._fields
xqueries = []
- for name, value in kw.iteritems():
+ for name, value in kw.items():
# If name is a field not yet indexed, return nothing
if name not in metadata:
warn_not_indexed(name)
diff --git a/itools/database/backends/git.py b/itools/database/backends/git.py
index cd5b2dc8a..77eeabaef 100644
--- a/itools/database/backends/git.py
+++ b/itools/database/backends/git.py
@@ -32,14 +32,17 @@
from itools.database.magic_ import magic_from_buffer
from itools.database.git import open_worktree
from itools.fs import lfs
+from itools.fs.common import WRITE, READ_WRITE, APPEND, READ
# Import from here
-from catalog import Catalog, _get_xquery, SearchResults, make_catalog
-from registry import register_backend
+from .catalog import Catalog, _get_xquery, SearchResults, make_catalog
+from .registry import register_backend
TEST_DB_WITHOUT_COMMITS = bool(int(os.environ.get('TEST_DB_WITHOUT_COMMITS') or 0))
TEST_DB_DESACTIVATE_GIT = bool(int(os.environ.get('TEST_DB_DESACTIVATE_GIT') or 0))
+TEST_DB_DESACTIVATE_STATIC_HISTORY = bool(int(os.environ.get('TEST_DB_DESACTIVATE_STATIC_HISTORY') or 1))
+TEST_DB_DESACTIVATE_PATCH = bool(int(os.environ.get('TEST_DESACTIVATE_PATCH') or 1))
class Heap(object):
@@ -67,15 +70,12 @@ def __init__(self):
self._dict = {}
self._heap = []
-
def __len__(self):
return len(self._dict)
-
def get(self, path):
return self._dict.get(path)
-
def __setitem__(self, path, value):
if path not in self._dict:
n = -path.count('/') if path else 1
@@ -83,14 +83,12 @@ def __setitem__(self, path, value):
self._dict[path] = value
-
def popitem(self):
key = heappop(self._heap)
path = key[1]
return path, self._dict.pop(path)
-
class GitBackend(object):
def __init__(self, path, fields, read_only=False):
@@ -107,7 +105,10 @@ def __init__(self, path, fields, read_only=False):
error = '"{0}" should be a folder, but it is not'.format(self.path_data)
raise ValueError(error)
# New interface to Git
- self.worktree = open_worktree(self.path_data)
+ if TEST_DB_DESACTIVATE_GIT is True:
+ self.worktree = None
+ else:
+ self.worktree = open_worktree(self.path_data)
# Initialize the database, but chrooted
self.fs = lfs.open(self.path_data)
# Static FS
@@ -118,7 +119,6 @@ def __init__(self, path, fields, read_only=False):
# Catalog
self.catalog = self.get_catalog()
-
@classmethod
def init_backend(cls, path, fields, init=False, soft=False):
# Metadata database
@@ -128,14 +128,12 @@ def init_backend(cls, path, fields, init=False, soft=False):
# Make catalog
make_catalog('{0}/catalog'.format(path), fields)
-
@classmethod
def init_backend_static(cls, path):
# Static database
lfs.make_folder('{0}/database_static'.format(path))
lfs.make_folder('{0}/database_static/.history'.format(path))
-
#######################################################################
# Database API
#######################################################################
@@ -148,56 +146,49 @@ def normalize_key(self, path, __root=None):
raise ValueError(err.format(path))
return '/'.join(key)
-
def handler_exists(self, key):
fs = self.get_handler_fs_by_key(key)
return fs.exists(key)
-
def get_handler_names(self, key):
return self.fs.get_names(key)
-
- def get_handler_data(self, key):
+ def get_handler_data(self, key, text=False):
if not key:
return None
fs = self.get_handler_fs_by_key(key)
- with fs.open(key) as f:
+ with fs.open(key, text=text) as f:
return f.read()
-
def get_handler_mimetype(self, key):
data = self.get_handler_data(key)
return magic_from_buffer(data)
-
def handler_is_file(self, key):
fs = self.get_handler_fs_by_key(key)
return fs.is_file(key)
-
def handler_is_folder(self, key):
fs = self.get_handler_fs_by_key(key)
return fs.is_folder(key)
-
def get_handler_mtime(self, key):
fs = self.get_handler_fs_by_key(key)
return fs.get_mtime(key)
-
def save_handler(self, key, handler):
data = handler.to_str()
+ text = isinstance(data, str)
# Save the file
fs = self.get_handler_fs(handler)
# Write and truncate (calls to "_save_state" must be done with the
# pointer pointing to the beginning)
if not fs.exists(key):
- with fs.make_file(key) as f:
+ with fs.make_file(key, text=text) as f:
f.write(data)
f.truncate(f.tell())
else:
- with fs.open(key, 'w') as f:
+ with fs.open(key, text=text, mode=READ_WRITE) as f:
f.write(data)
f.truncate(f.tell())
# Set dirty = None
@@ -205,23 +196,19 @@ def save_handler(self, key, handler):
handler.dirty = None
-
def traverse_resources(self):
raise NotImplementedError
-
def get_handler_fs(self, handler):
if isinstance(handler, Metadata):
return self.fs
return self.static_fs
-
def get_handler_fs_by_key(self, key):
if key.endswith('metadata'):
return self.fs
return self.static_fs
-
def add_handler_into_static_history(self, key):
the_time = datetime.now().strftime('%Y%m%d%H%M%S')
new_key = '.history/{0}.{1}.{2}'.format(key, the_time, uuid4())
@@ -230,12 +217,13 @@ def add_handler_into_static_history(self, key):
self.static_fs.make_folder(parent_path)
self.static_fs.copy(key, new_key)
-
def create_patch(self, added, changed, removed, handlers, git_author):
""" We create a patch into database/.git/patchs at each transaction.
The idea is to commit into GIT each N transactions on big databases to avoid performances problems.
We want to keep a diff on each transaction, to help debug.
"""
+ if TEST_DB_DESACTIVATE_PATCH is True:
+ return
author_id, author_email = git_author
diffs = {}
# Added
@@ -276,17 +264,17 @@ def create_patch(self, added, changed, removed, handlers, git_author):
f.write(data)
f.truncate(f.tell())
-
def do_transaction(self, commit_message, data, added, changed, removed, handlers,
docs_to_index, docs_to_unindex):
git_author, git_date, git_msg, docs_to_index, docs_to_unindex = data
# Statistics
self.nb_transactions += 1
# Add static changed & removed files to ~/database_static/.history/
- changed_and_removed = list(changed) + list(removed)
- for key in changed_and_removed:
- if not key.endswith('metadata'):
- self.add_handler_into_static_history(key)
+ if TEST_DB_DESACTIVATE_STATIC_HISTORY is False:
+ changed_and_removed = list(changed) + list(removed)
+ for key in changed_and_removed:
+ if not key.endswith('metadata'):
+ self.add_handler_into_static_history(key)
# Create patch if there's changed
if added or changed or removed:
self.create_patch(added, changed, removed, handlers, git_author)
@@ -322,7 +310,6 @@ def do_transaction(self, commit_message, data, added, changed, removed, handlers
self.catalog.index_document(values)
self.catalog.save_changes()
-
def do_git_big_commit(self):
""" Some databases are really bigs (1 millions files). GIT is too slow in this cases.
So we don't commit at each transaction, but at each N transactions.
@@ -333,28 +320,25 @@ def do_git_big_commit(self):
p1.start()
self.last_transaction_dtime = datetime.now()
-
def _do_git_big_commit(self):
- worktree = self.worktree
- worktree._call(['git', 'add', '-A'])
- worktree._call(['git', 'commit', '-m', 'Autocommit'])
-
+ self.worktree._call(['git', 'add', '-A'])
+ self.worktree._call(['git', 'commit', '-m', 'Autocommit'])
def do_git_transaction(self, commit_message, data, added, changed, removed, handlers):
worktree = self.worktree
# 3. Git add
git_add = list(added) + list(changed)
git_add = [x for x in git_add if x.endswith('metadata')]
- worktree.git_add(*git_add)
+ self.worktree.git_add(*git_add)
# 3. Git rm
git_rm = list(removed)
git_rm = [x for x in git_rm if x.endswith('metadata')]
- worktree.git_rm(*git_rm)
+ self.worktree.git_rm(*git_rm)
# 2. Build the 'git commit' command
git_author, git_date, git_msg, docs_to_index, docs_to_unindex = data
git_msg = git_msg or 'no comment'
# 4. Create the tree
- repo = worktree.repo
+ repo = self.worktree.repo
index = repo.index
try:
head = repo.revparse_single('HEAD')
@@ -414,8 +398,7 @@ def do_git_transaction(self, commit_message, data, added, changed, removed, hand
else:
tb.insert(name, value[0], value[1])
# 5. Git commit
- worktree.git_commit(git_msg, git_author, git_date, tree=git_tree)
-
+ self.worktree.git_commit(git_msg, git_author, git_date, tree=git_tree)
def abort_transaction(self):
self.catalog.abort_changes()
@@ -426,13 +409,12 @@ def abort_transaction(self):
#else:
# self.worktree.repo.checkout_head(strategy)
-
def flush_catalog(self, docs_to_unindex, docs_to_index):
for path in docs_to_unindex:
self.catalog.unindex_document(path)
for resource, values in docs_to_index:
self.catalog.index_document(values)
-
+ self.catalog.save_changes()
def get_catalog(self):
path = '{0}/catalog'.format(self.path)
@@ -440,7 +422,6 @@ def get_catalog(self):
return None
return Catalog(path, self.fields, read_only=self.read_only)
-
def search(self, query=None, **kw):
"""Launch a search in the catalog.
"""
@@ -448,7 +429,6 @@ def search(self, query=None, **kw):
xquery = _get_xquery(catalog, query, **kw)
return SearchResults(catalog, xquery)
-
def close(self):
self.catalog.close()
diff --git a/itools/database/backends/lfs.py b/itools/database/backends/lfs.py
index 82174b2e6..957c3705a 100644
--- a/itools/database/backends/lfs.py
+++ b/itools/database/backends/lfs.py
@@ -16,12 +16,13 @@
# Import from the Standard Library
from os.path import abspath, dirname
+import mimetypes
# Import from itools
-from itools.fs import lfs
+from itools.fs import lfs, WRITE
# Import from here
-from registry import register_backend
+from .registry import register_backend
class LFSBackend(object):
@@ -32,64 +33,56 @@ def __init__(self, path, fields, read_only=False):
else:
self.fs = lfs
-
@classmethod
def init_backend(cls, path, fields, init=False, soft=False):
- self.fs.make_folder('{0}/database'.format(path))
-
+ lfs.make_folder('{0}/database'.format(path))
def normalize_key(self, path, __root=None):
return self.fs.normalize_key(path)
-
def handler_exists(self, key):
return self.fs.exists(key)
-
def get_handler_names(self, key):
return self.fs.get_names(key)
-
- def get_handler_data(self, key):
+ def get_handler_data(self, key, text=False):
if not key:
return None
- with self.fs.open(key) as f:
- return f.read()
-
+ with self.fs.open(key, text=text) as f:
+ if isinstance(f, str):
+ return f
+ else:
+ return f.read()
def get_handler_mimetype(self, key):
return self.fs.get_mimetype(key)
-
def handler_is_file(self, key):
return self.fs.is_file(key)
-
def handler_is_folder(self, key):
return self.fs.is_folder(key)
-
def get_handler_mtime(self, key):
return self.fs.get_mtime(key)
-
def save_handler(self, key, handler):
data = handler.to_str()
+ text = isinstance(data, str)
# Save the file
if not self.fs.exists(key):
- with self.fs.make_file(key) as f:
+ with self.fs.make_file(key, text=text) as f:
f.write(data)
f.truncate(f.tell())
else:
- with self.fs.open(key, 'w') as f:
+ with self.fs.open(key, WRITE, text=text) as f:
f.write(data)
f.truncate(f.tell())
-
def traverse_resources(self):
raise NotImplementedError
-
def do_transaction(self, commit_message, data, added, changed, removed, handlers):
# List of Changed
added_and_changed = list(added) + list(changed)
@@ -103,11 +96,12 @@ def do_transaction(self, commit_message, data, added, changed, removed, handlers
for key in removed:
self.fs.remove(key)
-
-
def abort_transaction(self):
# Cannot abort transaction with this backend
pass
+ def close(self):
+ self.fs.close()
+
register_backend('lfs', LFSBackend)
diff --git a/itools/database/backends/registry.py b/itools/database/backends/registry.py
index b5f4055a7..83456c3d0 100644
--- a/itools/database/backends/registry.py
+++ b/itools/database/backends/registry.py
@@ -16,7 +16,6 @@
# Import from the Standard Library
-
backends_registry = {}
def register_backend(name, backends_cls):
diff --git a/itools/database/exceptions.py b/itools/database/exceptions.py
index 070e620e9..197a82e37 100644
--- a/itools/database/exceptions.py
+++ b/itools/database/exceptions.py
@@ -15,6 +15,6 @@
# along with this program. If not, see .
-class ReadonlyError(StandardError):
+class ReadonlyError(Exception):
pass
diff --git a/itools/database/fields.py b/itools/database/fields.py
index 642112c97..841ab056a 100644
--- a/itools/database/fields.py
+++ b/itools/database/fields.py
@@ -30,22 +30,19 @@ class Field(prototype):
multiple = False
empty_values = (None, '', [], (), {})
base_error_messages = {
- 'invalid': MSG(u'Invalid value.'),
- 'required': MSG(u'This field is required.'),
+ 'invalid': MSG('Invalid value.'),
+ 'required': MSG('This field is required.'),
}
error_messages = {}
validators = []
-
def get_datatype(self):
return self.datatype
-
def access(self, mode, resource):
# mode may be "read" or "write"
return True
-
def get_validators(self):
validators = []
for v in self.validators:
@@ -54,7 +51,6 @@ def get_validators(self):
validators.append(v)
return validators
-
def get_error_message(self, code):
messages = merge_dicts(
self.base_error_messages,
@@ -62,7 +58,6 @@ def get_error_message(self, code):
return messages.get(code)
-
def get_field_and_datatype(elt):
""" Now schema can be Datatype or Field.
To be compatible:
diff --git a/itools/database/git.py b/itools/database/git.py
index b0c38f9d8..1ed5df1cc 100644
--- a/itools/database/git.py
+++ b/itools/database/git.py
@@ -54,7 +54,6 @@ def make_parent_dirs(path):
makedirs(folder)
-
class Worktree(object):
def __init__(self, path, repo):
@@ -69,11 +68,11 @@ def __init__(self, path, repo):
try:
_, _ = self.username, self.useremail
except:
- print '========================================='
- print 'ERROR: Please configure GIT commiter via'
- print ' $ git config --global user.name'
- print ' $ git config --global user.email'
- print '========================================='
+ print('========================================')
+ print('ERROR: Please configure GIT commiter via')
+ print(' $ git config --global user.name')
+ print(' $ git config --global user.email')
+ print('=========================================')
raise
@@ -88,12 +87,11 @@ def _get_abspath(self, path):
if isabs(path):
if path.startswith(self.path):
return path
- raise ValueError, 'unexpected absolute path "%s"' % path
+ raise ValueError("unexpected absolute path '{}'".format(path))
if path == '.':
return self.path
return '%s%s' % (self.path, path)
-
def _call(self, command):
"""Interface to cal git.git for functions not yet implemented using
libgit2.
@@ -101,10 +99,9 @@ def _call(self, command):
popen = Popen(command, stdout=PIPE, stderr=PIPE, cwd=self.path)
stdoutdata, stderrdata = popen.communicate()
if popen.returncode != 0:
- raise EnvironmentError, (popen.returncode, stderrdata)
+ raise EnvironmentError((popen.returncode, stderrdata))
return stdoutdata
-
def _resolve_reference(self, reference):
"""This method returns the SHA the given reference points to. For now
only HEAD is supported.
@@ -125,7 +122,6 @@ def _resolve_reference(self, reference):
return reference.target
-
#######################################################################
# External API
#######################################################################
@@ -144,13 +140,13 @@ def walk(self, path='.'):
"""
# 1. Check and normalize path
if isabs(path):
- raise ValueError, 'unexpected absolute path "%s"' % path
+ raise ValueError('unexpected absolute path "%s"' % path)
path = normpath(path)
if path == '.':
path = ''
elif path == '.git':
- raise ValueError, 'cannot walk .git'
+ raise ValueError('cannot walk .git')
elif not isdir('%s%s' % (self.path, path)):
yield path
return
@@ -173,7 +169,6 @@ def walk(self, path='.'):
yield path_rel
-
def lookup(self, sha):
"""Return the object by the given SHA. We use a cache to warrant that
two calls with the same SHA will resolve to the same object, so the
@@ -185,7 +180,6 @@ def lookup(self, sha):
return cache[sha]
-
def lookup_from_commit_by_path(self, commit, path):
"""Return the object (tree or blob) the given path points to from the
given commit, or None if the given path does not exist.
@@ -204,7 +198,6 @@ def lookup_from_commit_by_path(self, commit, path):
obj = self.lookup(entry.oid)
return obj
-
@property
def index(self):
"""Gives access to the index file. Reloads the index file if it has
@@ -216,7 +209,7 @@ def index(self):
index = self.repo.index
# Bare repository
if index is None:
- raise RuntimeError, 'expected standard repository, not bare'
+ raise RuntimeError('expected standard repository, not bare')
path = self.index_path
if exists(path):
@@ -227,7 +220,6 @@ def index(self):
return index
-
def update_tree_cache(self):
"""libgit2 is able to read the tree cache, but not to write it.
To speed up 'git_commit' this method should be called from time to
@@ -236,7 +228,6 @@ def update_tree_cache(self):
command = ['git', 'write-tree']
self._call(command)
-
def git_add(self, *args):
"""Equivalent 'git add', adds the given paths to the index file.
If a path is a folder, adds all its content recursively.
@@ -247,7 +238,6 @@ def git_add(self, *args):
if path[-1] != '/':
index.add(path)
-
def git_rm(self, *args):
"""Equivalent to 'git rm', removes the given paths from the index
file and from the filesystem. If a path is a folder removes all
@@ -269,7 +259,6 @@ def git_rm(self, *args):
remove('%s/%s' % (root, name))
rmdir(root)
-
def git_mv(self, source, target, add=True):
"""Equivalent to 'git mv': moves the file or folder in the filesystem
from 'source' to 'target', removes the source from the index file,
@@ -296,7 +285,6 @@ def git_mv(self, source, target, add=True):
if add is True:
self.git_add(target)
-
@lazy
def username(self):
cmd = ['git', 'config', '--get', 'user.name']
@@ -306,7 +294,6 @@ def username(self):
raise ValueError("Please configure 'git config --global user.name'")
return username
-
@lazy
def useremail(self):
cmd = ['git', 'config', '--get', 'user.email']
@@ -316,7 +303,6 @@ def useremail(self):
raise ValueError("Please configure 'git config --global user.email'")
return useremail
-
def git_tag(self, tag_name, message):
"""Equivalent to 'git tag', we must give the name of the tag and the message
TODO Implement using libgit2
@@ -326,14 +312,12 @@ def git_tag(self, tag_name, message):
cmd = ['git', 'tag', '-a', tag_name, '-m', message]
return self._call(cmd)
-
def git_remove_tag(self, tag_name):
if not tag_name:
raise ValueError('excepted tag name')
cmd = ['git', 'tag', '-d', tag_name]
return self._call(cmd)
-
def git_reset(self, reference):
"""Equivalent to 'git reset --hard', we must provide the reference to reset to
"""
@@ -342,7 +326,6 @@ def git_reset(self, reference):
cmd = ['git', 'reset', '--hard', '-q', reference]
return self._call(cmd)
-
def git_commit(self, message, author=None, date=None, tree=None):
"""Equivalent to 'git commit', we must give the message and we can
also give the author and date.
@@ -368,7 +351,12 @@ def git_commit(self, message, author=None, date=None, tree=None):
name = self.username
email = self.useremail
- committer = Signature(name, email, when_time, when_offset)
+ if not isinstance(name, str):
+ name = name.decode("utf-8")
+ if not isinstance(email, str):
+ email = email.decode("utf-8")
+
+ committer = Signature(name, email, int(when_time), int(when_offset))
# Author
if author is None:
@@ -383,15 +371,14 @@ def git_commit(self, message, author=None, date=None, tree=None):
when_offset = date.utcoffset().seconds / 60
else:
err = "Worktree.git_commit doesn't support naive datatime yet"
- raise NotImplementedError, err
+ raise NotImplementedError(err)
- author = Signature(author[0], author[1], when_time, when_offset)
+ author = Signature(author[0], author[1], int(when_time), int(when_offset))
# Create the commit
return self.repo.create_commit('HEAD', author, committer, message,
tree, parents)
-
def git_log(self, paths=None, n=None, author=None, grep=None,
reverse=False, reference='HEAD'):
"""Equivalent to 'git log', optional keyword parameters are:
@@ -458,7 +445,6 @@ def git_log(self, paths=None, n=None, author=None, grep=None,
# Ok
return commits
-
def git_diff(self, since, until=None, paths=None):
"""Return the diff between two commits, eventually reduced to the
given paths.
@@ -476,7 +462,6 @@ def git_diff(self, since, until=None, paths=None):
cmd.extend(paths)
return self._call(cmd)
-
def git_stats(self, since, until=None, paths=None):
"""Return statistics of the changes done between two commits,
eventually reduced to the given paths.
@@ -494,7 +479,6 @@ def git_stats(self, since, until=None, paths=None):
cmd.extend(paths)
return self._call(cmd)
-
def get_files_changed(self, since, until):
"""Return the files that have been changed between two commits.
@@ -504,8 +488,7 @@ def get_files_changed(self, since, until):
cmd = ['git', 'show', '--numstat', '--pretty=format:', expr]
data = self._call(cmd)
lines = data.splitlines()
- return frozenset([ line.split('\t')[-1] for line in lines if line ])
-
+ return frozenset([line.split('\t')[-1] for line in lines if line])
def get_metadata(self, reference='HEAD'):
"""Resolves the given reference and returns metadata information
@@ -532,7 +515,6 @@ def get_metadata(self, reference='HEAD'):
}
-
def open_worktree(path, init=False, soft=False):
try:
if init:
diff --git a/itools/database/metadata.py b/itools/database/metadata.py
index 96989eb4e..fd81a1e35 100644
--- a/itools/database/metadata.py
+++ b/itools/database/metadata.py
@@ -15,17 +15,19 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+from logging import getLogger
+
# Import from itools
from itools.core import add_type, freeze
from itools.datatypes import String
from itools.handlers import File, register_handler_class
-from itools.log import log_warning
# Import from here
-from fields import Field
-from metadata_parser import parse_table, MetadataProperty, property_to_str
-from metadata_parser import deserialize_parameters
+from .fields import Field
+from .metadata_parser import parse_table, MetadataProperty, property_to_str
+from .metadata_parser import deserialize_parameters
+log = getLogger("itools.database")
class DefaultField(Field):
@@ -35,7 +37,7 @@ class DefaultField(Field):
parameters_schema = freeze({})
parameters_schema_default = None
multilingual = False
-
+ encrypted = False
class Metadata(File):
@@ -50,7 +52,6 @@ def reset(self):
self.version = None
self.properties = {}
-
def __init__(self, key=None, string=None, database=None, cls=None, **kw):
self.cls = cls
self.database = database
@@ -58,42 +59,38 @@ def __init__(self, key=None, string=None, database=None, cls=None, **kw):
proxy = super(Metadata, self)
proxy.__init__(key=key, string=string, database=database, **kw)
-
def new(self, cls=None, format=None, version=None):
self.cls = cls
self.format = format or cls.class_id
self.version = version or cls.class_version
-
def get_resource_class(self, class_id):
if self.cls:
return self.cls
return self.database.get_resource_class(class_id)
-
def change_class_id(self, new_class_id):
self.cls = None
self.set_changed()
self.format = new_class_id
self.get_resource_class(new_class_id)
-
def _load_state_from_file(self, file):
properties = self.properties
data = file.read()
parser = parse_table(data)
# Read the format & version
- name, value, parameters = parser.next()
+ name, value, parameters = next(parser)
if name != 'format':
- raise ValueError, 'unexpected "%s" property' % name
+ raise ValueError('unexpected "%s" property' % name)
if 'version' in parameters:
version = parameters.pop('version')
if len(version) > 1:
- raise ValueError, 'version parameter cannot be repeated'
+ raise ValueError('version parameter cannot be repeated')
self.version = version[0]
if parameters:
- raise ValueError, 'unexpected parameters for the format property'
+ raise ValueError('unexpected parameters for the format property')
self.format = value
# Get the schema
resource_class = self.get_resource_class(self.format)
@@ -101,7 +98,7 @@ def _load_state_from_file(self, file):
# Parse
for name, value, parameters in parser:
if name == 'format':
- raise ValueError, 'unexpected "format" property'
+ raise ValueError('unexpected "format" property')
# 1. Get the field
field = resource_class.get_field(name)
@@ -109,10 +106,10 @@ def _load_state_from_file(self, file):
msg = 'unexpected field "{0}" in resource {1}, cls {2}'
msg = msg.format(name, self.key, resource_class)
if resource_class.fields_soft:
- log_warning(msg, domain='itools.database')
+ log.warning(msg)
field = DefaultField
else:
- raise ValueError, msg
+ raise ValueError(msg)
# 2. Deserialize the parameters
params_schema = field.parameters_schema
@@ -120,17 +117,20 @@ def _load_state_from_file(self, file):
try:
deserialize_parameters(parameters, params_schema,
params_default)
- except ValueError, e:
+ except ValueError as e:
msg = 'in class "{0}", resource {1} property "{2}": {3}'
- raise ValueError, msg.format(resource_class, self.key, name, e)
+ raise ValueError(msg.format(resource_class, self.key, name, e))
# 3. Get the datatype properties
if field.multiple and field.multilingual:
error = 'property "%s" is both multilingual and multiple'
- raise ValueError, error % name
+ raise ValueError(error % name)
# 4. Build the property
datatype = field.datatype
+ datatype.encrypted = field.encrypted
+ if datatype.encrypted:
+ value = datatype.decrypt(value)
property = MetadataProperty(value, datatype, **parameters)
# Case 1: Multilingual
@@ -148,7 +148,6 @@ def _load_state_from_file(self, file):
else:
properties[name] = property
-
def to_str(self):
resource_class = self.get_resource_class(self.format)
@@ -158,8 +157,7 @@ def to_str(self):
lines = ['format;version=%s:%s\n' % (self.version, self.format)]
# Properties are to be sorted by alphabetical order
properties = self.properties
- names = properties.keys()
- names.sort()
+ names = sorted(list(properties.keys()))
# Properties
for name in names:
@@ -171,17 +169,17 @@ def to_str(self):
msg = 'unexpected field "{0}" in resource "{1}" (format "{2}")'
msg = msg.format(name, self.key, self.format)
if resource_class.fields_soft:
- log_warning(msg, domain='itools.database')
+ log.warning(msg)
continue
- raise ValueError, msg
-
+ raise ValueError(msg)
datatype = field.datatype
+ datatype.encrypted = field.encrypted
params_schema = field.parameters_schema
is_empty = datatype.is_empty
p_type = type(property)
if p_type is dict:
- languages = property.keys()
- languages.sort()
+ languages = list(property.keys())
+ languages = sorted(languages)
lines += [
property_to_str(name, property[x], datatype, params_schema)
for x in languages if not is_empty(property[x].value) ]
@@ -197,7 +195,6 @@ def to_str(self):
return ''.join(lines)
-
########################################################################
# API
########################################################################
@@ -241,7 +238,6 @@ def get_property(self, name, language=None):
return property[language]
-
def has_property(self, name, language=None):
if name not in self.properties:
return False
@@ -251,7 +247,6 @@ def has_property(self, name, language=None):
return True
-
def _set_property(self, name, value):
properties = self.properties
@@ -290,12 +285,10 @@ def _set_property(self, name, value):
if not field.datatype.is_empty(value.value):
properties.setdefault(name, []).append(value)
-
def set_property(self, name, value):
self.set_changed()
self._set_property(name, value)
-
def del_property(self, name):
if name in self.properties:
self.set_changed()
diff --git a/itools/database/metadata_parser.py b/itools/database/metadata_parser.py
old mode 100644
new mode 100755
index 3689a8d62..8b41b0ff4
--- a/itools/database/metadata_parser.py
+++ b/itools/database/metadata_parser.py
@@ -35,6 +35,20 @@
('\r', r'\r'),
('\n', r'\n'))
+def decode_lines(lines):
+ new_lines = []
+ for line in lines:
+ if type(line) is bytes:
+ for encoding in ["utf-8", "latin-1"]:
+ try:
+ line = line.decode(encoding)
+ break
+ except:
+ pass
+ if type(line) is bytes:
+ raise Exception("Error decoding lines")
+ new_lines.append(line)
+ return new_lines
def unescape_data(data, escape_table=escape_table):
"""Unescape the data
@@ -48,7 +62,6 @@ def unescape_data(data, escape_table=escape_table):
return '\\'.join(out)
-
def escape_data(data, escape_table=escape_table):
"""Escape the data
"""
@@ -58,13 +71,12 @@ def escape_data(data, escape_table=escape_table):
return data
-
def unfold_lines(data):
"""Unfold the folded lines.
"""
i = 0
lines = data.splitlines()
-
+ lines = decode_lines(lines)
line = ''
while i < len(lines):
next = lines[i]
@@ -79,7 +91,6 @@ def unfold_lines(data):
yield line
-
def fold_line(data):
"""Fold the unfolded line over 75 characters.
"""
@@ -104,7 +115,6 @@ def fold_line(data):
return res
-
# XXX The RFC only allows '-', we allow more because this is used by
# itools.database (metadata). This set matches checkid (itools.handlers)
allowed = frozenset(['-', '_', '.', '@'])
@@ -118,7 +128,7 @@ def read_name(line, allowed=allowed):
# Test first character of name
c = line[0]
if not c.isalnum() and c != '-':
- raise SyntaxError, 'unexpected character (%s)' % c
+ raise SyntaxError('unexpected character (%s)' % c)
# Test the rest
idx = 1
@@ -130,9 +140,9 @@ def read_name(line, allowed=allowed):
if c.isalnum() or c in allowed:
idx += 1
continue
- raise SyntaxError, "unexpected character '%s' (%s)" % (c, ord(c))
+ raise SyntaxError("unexpected character '%s' (%s)" % (c, ord(c)))
- raise SyntaxError, 'unexpected end of line (%s)' % line
+ raise SyntaxError('unexpected end of line (%s)' % line)
# Manage an icalendar content line value property [with parameters] :
@@ -175,7 +185,7 @@ def get_tokens(property):
if c.isalnum() or c in ('-'):
param_name, status = c, 2
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
# param-name begun
elif status == 2:
@@ -185,7 +195,7 @@ def get_tokens(property):
parameters[param_name] = []
status = 3
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
# param-name ended, param-value beginning
elif status == 3:
@@ -230,7 +240,7 @@ def get_tokens(property):
parameters[param_name].append(param_value)
status = 3
elif c == '"':
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
else:
param_value += c
@@ -248,17 +258,16 @@ def get_tokens(property):
elif c == ',':
status = 3
else:
- raise SyntaxError, error1 % (c, status)
+ raise SyntaxError(error1 % (c, status))
if status not in (7, 8):
- raise SyntaxError, 'unexpected property (%s)' % property
+ raise SyntaxError('unexpected property (%s)' % property)
# Unescape special characters (TODO Check the spec)
value = unescape_data(value)
return value, parameters
-
def parse_table(data):
"""This is the public interface of the module "itools.ical.parser", a
low-level parser of iCalendar files.
@@ -278,7 +287,6 @@ def parse_table(data):
yield name, value, parameters
-
###########################################################################
# Helper functions
###########################################################################
@@ -294,7 +302,7 @@ def deserialize_parameters(parameters, schema, default=String(multiple=True)):
for name in parameters:
datatype = schema.get(name, default)
if datatype is None:
- raise ValueError, 'parameter "{0}" not defined'.format(name)
+ raise ValueError('parameter "{0}" not defined'.format(name))
# Decode
value = parameters[name]
value = [ decode_param_value(x, datatype) for x in value ]
@@ -302,13 +310,12 @@ def deserialize_parameters(parameters, schema, default=String(multiple=True)):
if not datatype.multiple:
if len(value) > 1:
msg = 'parameter "%s" must be a singleton'
- raise ValueError, msg % name
+ raise ValueError(msg % name)
value = value[0]
# Update
parameters[name] = value
-
class MetadataProperty(object):
"""A property has a value, and may have one or more parameters.
@@ -329,29 +336,25 @@ def value(self):
return self.datatype.decode(self.raw_value)
return self.raw_value
-
def clone(self):
# Copy the value and parameters
value = deepcopy(self.value)
parameters = {}
- for p_key, p_value in self.parameters.iteritems():
+ for p_key, p_value in self.parameters.items():
c_value = deepcopy(p_value)
parameters[p_key] = c_value
return MetadataProperty(value, self.datatype, **parameters)
-
def get_parameter(self, name, default=None):
if self.parameters is None:
return default
return self.parameters.get(name, default)
-
def set_parameter(self, name, value):
if self.parameters is None:
self.parameters = {}
self.parameters[name] = value
-
def __eq__(self, other):
if type(other) is not MetadataProperty:
return False
@@ -359,14 +362,10 @@ def __eq__(self, other):
return False
return self.parameters == other.parameters
-
def __ne__(self, other):
return not self.__eq__(other)
-
-
-
params_escape_table = (
('"', r'\"'),
('\r', r'\r'),
@@ -384,7 +383,7 @@ def encode_param_value(p_name, p_value, p_datatype):
# Standard case (ical behavior)
if '"' in p_value or '\n' in p_value:
error = 'the "%s" parameter contains a double quote'
- raise ValueError, error % p_name
+ raise ValueError(error % p_name)
if ';' in p_value or ':' in p_value or ',' in p_value:
return '"%s"' % p_value
return p_value
@@ -408,8 +407,8 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
"""
# Parameters
if property.parameters:
- p_names = property.parameters.keys()
- p_names.sort()
+ p_names = list(property.parameters.keys())
+ p_names = sorted(p_names)
else:
p_names = []
@@ -439,9 +438,11 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
else:
value = datatype.encode(property.value)
if type(value) is not str:
- raise ValueError, 'property "{0}" is not str but {1}'.format(
- name, type(value))
+ raise ValueError('property "{0}" is not str but {1}'.format(
+ name, type(value)))
value = escape_data(value)
+ if datatype.encrypted:
+ value = datatype.encrypt(value)
# Ok
property = '%s%s:%s\n' % (name, parameters, value)
@@ -451,6 +452,6 @@ def _property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
def property_to_str(name, property, datatype, p_schema, encoding='utf-8'):
try:
return _property_to_str(name, property, datatype, p_schema, encoding)
- except StandardError:
+ except Exception:
err = 'failed to serialize "%s" property, probably a bad value'
- raise ValueError, err % name
+ raise ValueError(err % name)
diff --git a/itools/database/queries.py b/itools/database/queries.py
index ca774998e..53952943a 100644
--- a/itools/database/queries.py
+++ b/itools/database/queries.py
@@ -42,7 +42,6 @@ def __repr__(self):
self.__repr_parameters__())
-
class AllQuery(BaseQuery):
def __repr_parameters__(self):
@@ -93,7 +92,6 @@ def append(self, atom):
raise NotImplementedError
-
class _AndQuery(_MultipleQuery):
def append(self, atom):
@@ -107,7 +105,6 @@ def append(self, atom):
atoms.append(atom)
-
class _OrQuery(_MultipleQuery):
def append(self, atom):
@@ -123,7 +120,6 @@ def append(self, atom):
atoms.append(atom)
-
def _flat_query(cls, *args):
query = cls()
for subquery in args:
@@ -135,52 +131,43 @@ def _flat_query(cls, *args):
return query
-
def AndQuery(*args):
return _flat_query(_AndQuery, *args)
-
def OrQuery(*args):
return _flat_query(_OrQuery, *args)
-
class NotQuery(BaseQuery):
def __init__(self, query):
self.query = query
-
def __repr_parameters__(self):
return repr(self.query)
-
class StartQuery(BaseQuery):
def __init__(self, name, value):
self.name = name
self.value = value
-
def __repr_parameters__(self):
return "%r, %r" % (self.name, self.value)
-
class TextQuery(BaseQuery):
def __init__(self, name, value):
self.name = name
self.value = value
-
def __repr_parameters__(self):
return "%r, %r" % (self.name, self.value)
-
class QueryPrinter(PrettyPrinter):
def _format(self, query, stream, indent, allowance, context, level):
diff --git a/itools/database/registry.py b/itools/database/registry.py
index b6f67b8c1..f380e24bf 100644
--- a/itools/database/registry.py
+++ b/itools/database/registry.py
@@ -17,10 +17,9 @@
# Import from the Standard Library
from types import MethodType
-
-
fields_registry = {}
+
def register_field(name, field_cls):
if name not in fields_registry:
fields_registry[name] = field_cls
@@ -38,9 +37,10 @@ def register_field(name, field_cls):
old_value = getattr(old, key, None)
new_value = getattr(new, key, None)
if type(old_value) is MethodType:
- old_value = old_value.im_func
+ old_value = old_value.__func__
if type(new_value) is MethodType:
- new_value = new_value.im_func
+ new_value = new_value.__func__
+
if old_value != new_value:
msg = 'register conflict over the "{0}" field ({1} is different)'
raise ValueError(msg.format(name, key))
diff --git a/itools/database/resources.py b/itools/database/resources.py
old mode 100644
new mode 100755
index c5a11713a..5a739aadc
--- a/itools/database/resources.py
+++ b/itools/database/resources.py
@@ -14,14 +14,16 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+from logging import getLogger
# Import from itools
from itools.core import freeze, is_prototype
# Import from itools.database
-from fields import Field
-from registry import register_field
-from ro import RODatabase
+from .fields import Field
+from .registry import register_field
+from .ro import RODatabase
+log = getLogger("itools.database")
class DBResourceMetaclass(type):
@@ -33,8 +35,7 @@ def __new__(mcs, name, bases, dict):
# Lookup fields
if 'fields' not in dict:
- cls.fields = [ x for x in dir(cls)
- if is_prototype(getattr(cls, x), Field) ]
+ cls.fields = [x for x in dir(cls) if is_prototype(getattr(cls, x), Field)]
# Register new fields in the catalog
for name in cls.fields:
@@ -48,13 +49,10 @@ def __new__(mcs, name, bases, dict):
return cls
+class Resource(object, metaclass=DBResourceMetaclass):
-class Resource(object):
-
- __metaclass__ = DBResourceMetaclass
__hash__ = None
-
fields = freeze([])
# Says what to do when a field not defined by the schema is found.
@@ -62,25 +60,21 @@ class Resource(object):
# soft = True : log a warning
fields_soft = False
-
@classmethod
def get_field(cls, name, soft=True):
if name in cls.fields:
return getattr(cls, name, None)
- msg = 'Undefined field %s on %s' % (name, cls)
if soft is True:
return None
- print('Warning: '+ msg)
+ log.warning("Warning: Undefined field {} on {}".format(name, cls))
return None
-
@classmethod
def get_fields(self):
for name in self.fields:
field = self.get_field(name)
yield name, field
-
def get_catalog_values(self):
"""Returns a dictionary with the values of the fields to be indexed.
"""
diff --git a/itools/database/ro.py b/itools/database/ro.py
old mode 100644
new mode 100755
index 03e35b932..a7c09f1c4
--- a/itools/database/ro.py
+++ b/itools/database/ro.py
@@ -28,10 +28,10 @@
from itools.uri import Path
# Import from itools.database
-from backends import GitBackend, backends_registry
-from exceptions import ReadonlyError
-from metadata import Metadata
-from registry import get_register_fields
+from .backends import GitBackend, backends_registry
+from .exceptions import ReadonlyError
+from .metadata import Metadata
+from .registry import get_register_fields
class SearchResults(object):
@@ -40,27 +40,22 @@ def __init__(self, database, results):
self.database = database
self.results = results
-
def __len__(self):
return len(self.results)
-
def search(self, query=None, **kw):
results = self.results.search(query, **kw)
return SearchResults(self.database, results)
-
def get_documents(self, sort_by=None, reverse=False, start=0, size=0):
return self.results.get_documents(sort_by, reverse, start, size)
-
def get_resources(self, sort_by=None, reverse=False, start=0, size=0):
brains = list(self.get_documents(sort_by, reverse, start, size))
for brain in brains:
yield self.database.get_resource_from_brain(brain)
-
class RODatabase(object):
read_only = True
@@ -79,21 +74,20 @@ def __init__(self, path=None, size_min=4800, size_max=5200, backend='lfs'):
# Fields
self.fields = get_register_fields()
# init backend
- self.backend = self.backend_cls(self.path, self.fields, self.read_only)
+ self.init_backend()
# A mapping from key to handler
self.cache = LRUCache(size_min, size_max, automatic=False)
+ def init_backend(self):
+ self.backend = self.backend_cls(self.path, self.fields, self.read_only)
@property
def catalog(self):
- print('WARNING: Uses of context.database.catalog is obsolete')
return self.backend.catalog
-
def close(self):
self.backend.close()
-
def check_database(self):
"""This function checks whether the database is in a consisitent state,
this is to say whether a transaction was not brutally aborted and left
@@ -101,11 +95,8 @@ def check_database(self):
This is meant to be used by scripts, like 'icms-start.py'
"""
- # TODO Check if bare repository is OK
- print('Checking database...')
return True
-
#######################################################################
# With statement
#######################################################################
@@ -113,7 +104,6 @@ def check_database(self):
def __enter__(self):
return self
-
def __exit__(self, exc_type, exc_value, traceback):
self.close()
@@ -141,7 +131,6 @@ def _sync_filesystem(self, key):
# Everything looks fine
return handler
-
def _discard_handler(self, key):
"""Unconditionally remove the handler identified by the given key from
the cache, and invalidate it (and free memory at the same time).
@@ -150,13 +139,11 @@ def _discard_handler(self, key):
# Invalidate the handler
handler.__dict__.clear()
-
def _abort_changes(self):
"""To be called to abandon the transaction.
"""
raise ReadonlyError
-
def _cleanup(self):
"""For maintenance operations, this method is automatically called
after a transaction is committed or aborted.
@@ -169,14 +156,12 @@ def _cleanup(self):
#print 'RODatabase._cleanup (1): % 4d %s' % (len(self.cache), vmsize())
#print gc.get_count()
-
#######################################################################
# Public API
#######################################################################
def normalize_key(self, path, __root=Path('/')):
return self.backend.normalize_key(path, __root)
-
def push_handler(self, key, handler):
"""Adds the given resource to the cache.
"""
@@ -192,7 +177,6 @@ def push_handler(self, key, handler):
if cache_is_full:
self.make_room()
-
def make_room(self):
"""Remove handlers from the cache until it fits the defined size.
@@ -222,8 +206,6 @@ def make_room(self):
if cache_is_full and n > 0:
self._discard_handler(key)
-
-
def has_handler(self, key):
key = self.normalize_key(key)
@@ -235,33 +217,26 @@ def has_handler(self, key):
# Ask backend
return self.backend.handler_exists(key)
-
def save_handler(self, key, handler):
self.backend.save_handler(key, handler)
-
def get_handler_names(self, key):
key = self.normalize_key(key)
return self.backend.get_handler_names(key)
-
- def get_handler_data(self, key):
- return self.backend.get_handler_data(key)
-
+ def get_handler_data(self, key, text=False):
+ return self.backend.get_handler_data(key, text=text)
def get_handler_mtime(self, key):
return self.backend.get_handler_mtime(key)
-
def get_mimetype(self, key):
return self.backend.get_handler_mimetype(key)
-
def get_handler_class(self, key):
mimetype = self.get_mimetype(key)
return get_handler_class_by_mimetype(mimetype)
-
def _get_handler(self, key, cls=None, soft=False):
# Get resource
if key in self.removed:
@@ -274,8 +249,7 @@ def _get_handler(self, key, cls=None, soft=False):
if handler is not None:
# Check the class matches
if cls is not None and not isinstance(handler, cls):
- error = "expected '%s' class, '%s' found"
- raise LookupError, error % (cls, handler.__class__)
+ raise LookupError("expected '{}' class, '{}' found".format(cls, handler.__class__))
# Cache hit
self.cache.touch(key)
return handler
@@ -309,22 +283,18 @@ def _get_handler(self, key, cls=None, soft=False):
# Ok
return handler
-
def traverse_resources(self):
return self.backend.traverse_resources()
-
def get_handler(self, key, cls=None, soft=False):
key = self.normalize_key(key)
return self._get_handler(key, cls, soft)
-
def get_handlers(self, key):
base = self.normalize_key(key)
for name in self.get_handler_names(base):
yield self._get_handler(base + '/' + name)
-
def touch_handler(self, key, handler=None):
"""Report a modification of the key/handler to the database.
"""
@@ -344,22 +314,17 @@ def touch_handler(self, key, handler=None):
# Set in changed list
self.changed.add(key)
-
def set_handler(self, key, handler):
- raise ReadonlyError, 'cannot set handler'
-
+ raise ReadonlyError('cannot set handler')
def del_handler(self, key):
- raise ReadonlyError, 'cannot del handler'
-
+ raise ReadonlyError('cannot del handler')
def copy_handler(self, source, target, exclude_patterns=None):
- raise ReadonlyError, 'cannot copy handler'
-
+ raise ReadonlyError('cannot copy handler')
def move_handler(self, source, target):
- raise ReadonlyError, 'cannot move handler'
-
+ raise ReadonlyError('cannot move handler')
#######################################################################
# Layer 1: resources
@@ -372,18 +337,16 @@ def register_resource_class(self, resource_class, format=None):
format = resource_class.class_id
self._resources_registry[format] = resource_class
-
@classmethod
def unregister_resource_class(self, resource_class):
registry = self._resources_registry
- for class_id, cls in registry.items():
+ for class_id, cls in list(registry.items()):
if resource_class is cls:
del registry[class_id]
-
def get_resource_class(self, class_id):
if type(class_id) is not str:
- raise TypeError, 'expected byte string, got %s' % class_id
+ raise TypeError('expected string, got {}'.format(class_id))
# Check dynamic models are not broken
registry = self._resources_registry
@@ -391,8 +354,7 @@ def get_resource_class(self, class_id):
model = self.get_resource(class_id, soft=True)
if model is None:
registry.pop(class_id, None)
- err = 'the resource "%s" does not exist' % class_id
- raise LookupError, err
+ raise LookupError("the resource '{}' does not exist".format(class_id))
# Cache hit
cls = registry.get(class_id)
@@ -415,10 +377,9 @@ def get_resource_class(self, class_id):
# Default
return self._resources_registry['application/octet-stream']
-
def get_resource_classes(self):
registry = self._resources_registry
- for class_id, cls in self._resources_registry.items():
+ for class_id, cls in list(self._resources_registry.items()):
if class_id[0] == '/':
model = self.get_resource(class_id, soft=True)
if model is None:
@@ -427,7 +388,6 @@ def get_resource_classes(self):
yield cls
-
def get_metadata(self, abspath, soft=False):
if type(abspath) is str:
path = abspath[1:]
@@ -437,12 +397,10 @@ def get_metadata(self, abspath, soft=False):
path_to_metadata = '%s.metadata' % path
return self.get_handler(path_to_metadata, Metadata, soft=soft)
-
def get_cls(self, class_id):
cls = self.get_resource_class(class_id)
return cls or self.get_resource_class('application/octet-stream')
-
def get_resource(self, abspath, soft=False):
abspath = Path(abspath)
# Get metadata
@@ -455,44 +413,34 @@ def get_resource(self, abspath, soft=False):
# Ok
return cls(abspath=abspath, database=self, metadata=metadata)
-
def get_resource_from_brain(self, brain):
cls = self.get_cls(brain.format)
return cls(abspath=Path(brain.abspath), database=self, brain=brain)
-
def remove_resource(self, resource):
- raise ReadonlyError
-
+ raise ReadonlyError
def add_resource(self, resource):
- raise ReadonlyError
-
+ raise ReadonlyError
def change_resource(self, resource):
- raise ReadonlyError
-
+ raise ReadonlyError
def move_resource(self, source, new_path):
- raise ReadonlyError
-
+ raise ReadonlyError
def save_changes(self):
return
-
def create_tag(self, tag_name, message=None):
raise ReadonlyError
-
def reset_to_tag(self, tag_name):
raise ReadonlyError
-
def abort_changes(self):
return
-
#######################################################################
# API for path
#######################################################################
@@ -502,14 +450,12 @@ def get_basename(path):
path = Path(path)
return path.get_name()
-
@staticmethod
def get_path(path):
if type(path) is not Path:
path = Path(path)
return str(path)
-
@staticmethod
def resolve(base, path):
if type(base) is not Path:
@@ -517,7 +463,6 @@ def resolve(base, path):
path = base.resolve(path)
return str(path)
-
@staticmethod
def resolve2(base, path):
if type(base) is not Path:
@@ -525,7 +470,6 @@ def resolve2(base, path):
path = base.resolve2(path)
return str(path)
-
#######################################################################
# Search
#######################################################################
@@ -533,10 +477,8 @@ def search(self, query=None, **kw):
results = self.backend.search(query, **kw)
return SearchResults(database=self, results=results)
-
def reindex_catalog(self, base_abspath, recursif=True):
raise ReadonlyError
-
ro_database = RODatabase()
diff --git a/itools/database/rw.py b/itools/database/rw.py
old mode 100644
new mode 100755
index 023cd3483..edb9f2fba
--- a/itools/database/rw.py
+++ b/itools/database/rw.py
@@ -21,17 +21,18 @@
# Import from the Standard Library
from datetime import datetime
import fnmatch
+from logging import getLogger
# Import from itools
from itools.fs import lfs
from itools.handlers import Folder
-from itools.log import log_error
# Import from here
-from backends import backends_registry
-from registry import get_register_fields
-from ro import RODatabase
+from .backends import backends_registry
+from .registry import get_register_fields
+from .ro import RODatabase
+log = getLogger("itools.database")
MSG_URI_IS_BUSY = 'The "%s" URI is busy.'
@@ -41,8 +42,7 @@ class RWDatabase(RODatabase):
read_only = False
def __init__(self, path, size_min, size_max, backend='git'):
- proxy = super(RWDatabase, self)
- proxy.__init__(path, size_min, size_max, backend)
+ super().__init__(path, size_min, size_max, backend)
# Changes on DB
self.added = set()
self.changed = set()
@@ -90,16 +90,13 @@ def __init__(self, path, size_min, size_max, backend='git'):
self.resources_old2new_catalog = {}
self.resources_new2old_catalog = {}
-
def check_catalog(self):
pass
-
def close(self):
self.abort_changes()
self.backend.close()
-
def _sync_filesystem(self, key):
# Don't check if handler has been modified since last loading,
# we only have one writer
@@ -121,7 +118,6 @@ def has_handler(self, key):
# Normal case
return super(RWDatabase, self).has_handler(key)
-
def _get_handler(self, key, cls=None, soft=False):
# A hook to handle the new directories
base = key + '/'
@@ -133,7 +129,6 @@ def _get_handler(self, key, cls=None, soft=False):
# The other files
return super(RWDatabase, self)._get_handler(key, cls, soft)
-
def set_handler(self, key, handler):
# TODO: We have to refactor the set_changed()
# mechanism in handlers/database
@@ -143,14 +138,14 @@ def set_handler(self, key, handler):
handler.loaded = True
handler.set_changed()
if type(handler) is Folder:
- raise ValueError, 'unexpected folder (only files can be "set")'
+ raise ValueError('unexpected folder (only files can be "set")')
if handler.key is not None:
- raise ValueError, 'only new files can be added, try to clone first'
+ raise ValueError('only new files can be added, try to clone first')
key = self.normalize_key(key)
if self._get_handler(key, soft=True) is not None:
- raise RuntimeError, MSG_URI_IS_BUSY % key
+ raise RuntimeError(MSG_URI_IS_BUSY % key)
# Added or modified ?
if key not in self.added and self.has_handler(key):
@@ -163,7 +158,6 @@ def set_handler(self, key, handler):
self.removed.discard(key)
self.has_changed = True
-
def del_handler(self, key):
key = self.normalize_key(key)
@@ -195,7 +189,6 @@ def del_handler(self, key):
self.removed.add(key)
self.has_changed = True
-
def touch_handler(self, key, handler=None):
key = self.normalize_key(key)
# Mark the handler as dirty
@@ -212,11 +205,9 @@ def touch_handler(self, key, handler=None):
self.changed.add(key)
-
def save_handler(self, key, handler):
self.backend.save_handler(key, handler)
-
def get_handler_names(self, key):
key = self.normalize_key(key)
# On the filesystem
@@ -234,7 +225,6 @@ def get_handler_names(self, key):
names.add(name)
return list(names)
-
def copy_handler(self, source, target, exclude_patterns=None):
source = self.normalize_key(source)
target = self.normalize_key(target)
@@ -252,7 +242,7 @@ def copy_handler(self, source, target, exclude_patterns=None):
# Check the target is free
if self._get_handler(target, soft=True) is not None:
- raise RuntimeError, MSG_URI_IS_BUSY % target
+ raise RuntimeError(MSG_URI_IS_BUSY % target)
handler = self._get_handler(source)
if type(handler) is Folder:
@@ -265,7 +255,6 @@ def copy_handler(self, source, target, exclude_patterns=None):
self.removed.discard(target)
self.has_changed = True
-
def move_handler(self, source, target):
source = self.normalize_key(source)
target = self.normalize_key(target)
@@ -276,7 +265,7 @@ def move_handler(self, source, target):
# Check the target is free
if self._get_handler(target, soft=True) is not None:
- raise RuntimeError, MSG_URI_IS_BUSY % target
+ raise RuntimeError(MSG_URI_IS_BUSY % target)
# Go
cache = self.cache
@@ -327,7 +316,6 @@ def move_handler(self, source, target):
self.removed.discard(target)
self.has_changed = True
-
#######################################################################
# Layer 1: resources
#######################################################################
@@ -342,7 +330,6 @@ def remove_resource(self, resource):
self.resources_old2new_catalog[path] = None
self.resources_new2old_catalog.pop(path, None)
-
def add_resource(self, resource):
self.has_changed = True
new2old = self.resources_new2old
@@ -351,7 +338,6 @@ def add_resource(self, resource):
path = str(x.abspath)
new2old[path] = None
-
def change_resource(self, resource):
self.has_changed = True
old2new = self.resources_old2new
@@ -362,14 +348,13 @@ def change_resource(self, resource):
return
# Case 2: removed or moved away
if path in old2new and not old2new[path]:
- raise ValueError, 'cannot change a resource that has been removed'
+ raise ValueError('cannot change a resource that has been removed')
# Case 3: not yet touched
old2new[path] = path
new2old[path] = path
self.resources_old2new_catalog[path] = path
self.resources_new2old_catalog[path] = path
-
def is_changed(self, resource):
"""We use for this function only the 2 dicts old2new and new2old.
"""
@@ -378,7 +363,6 @@ def is_changed(self, resource):
path = str(resource.abspath)
return path in old2new or path in new2old
-
def move_resource(self, source, new_path):
self.has_changed = True
old2new = self.resources_old2new
@@ -393,7 +377,7 @@ def move_resource(self, source, new_path):
target_path = str(target_path)
if source_path in old2new and not old2new[source_path]:
err = 'cannot move a resource that has been removed'
- raise ValueError, err
+ raise ValueError(err)
source_path = new2old.pop(source_path, source_path)
if source_path:
@@ -402,7 +386,6 @@ def move_resource(self, source, new_path):
new2old[target_path] = source_path
self.resources_new2old_catalog[target_path] = source_path
-
#######################################################################
# Transactions
#######################################################################
@@ -410,14 +393,13 @@ def _cleanup(self):
super(RWDatabase, self)._cleanup()
self.has_changed = False
-
def _abort_changes(self):
# 1. Handlers
cache = self.cache
for key in self.added:
self._discard_handler(key)
for key in self.changed:
- if cache.has_key(key):
+ if key in cache:
# FIXME
# We check cache since an handler
# can be added & changed at one
@@ -438,7 +420,6 @@ def _abort_changes(self):
self.resources_old2new_catalog.clear()
self.resources_new2old_catalog.clear()
-
def abort_changes(self):
if not self.has_changed:
return
@@ -446,7 +427,6 @@ def abort_changes(self):
self._abort_changes()
self._cleanup()
-
def _before_commit(self):
"""This method is called before 'save_changes', and gives a chance
to the database to check for preconditions, if an error occurs here
@@ -457,7 +437,6 @@ def _before_commit(self):
"""
return None, None, None, [], []
-
def _save_changes(self, data, commit_msg=None):
# Get data informations
the_author, the_date, the_msg, docs_to_index, docs_to_unindex = data
@@ -475,7 +454,6 @@ def _save_changes(self, data, commit_msg=None):
self.added.clear()
self.removed.clear()
-
def save_changes(self, commit_message=None):
if not self.has_changed:
return
@@ -483,12 +461,12 @@ def save_changes(self, commit_message=None):
# the transaction will be aborted
try:
data = self._before_commit()
- except Exception:
- log_error('Transaction failed', domain='itools.database')
+ except Exception as e:
+ log.error("Transaction failed", exc_info=True)
try:
self._abort_changes()
- except Exception:
- log_error('Aborting failed', domain='itools.database')
+ except Exception as e:
+ log.error("Aborting failed", exc_info=True)
self._cleanup()
raise
@@ -496,35 +474,31 @@ def save_changes(self, commit_message=None):
try:
self._save_changes(data, commit_message)
except Exception as e:
- log_error('Transaction failed', domain='itools.database')
+ log.error("Transaction failed", exc_info=True)
try:
self._abort_changes()
- except Exception:
- log_error('Aborting failed', domain='itools.database')
- raise(e)
+ except Exception as e:
+ log.error("Aborting failed", exc_info=True)
+ raise e
finally:
self._cleanup()
-
def flush_catalog(self):
""" Flush changes in catalog without commiting
(allow to search in catalog on changed elements)
"""
- root = self.get_resource('/')
- docs_to_index = set(self.resources_new2old_catalog.keys())
- docs_to_unindex = self.resources_old2new_catalog.keys()
- docs_to_unindex = list(set(docs_to_unindex) - docs_to_index)
- docs_to_index = list(docs_to_index)
- aux = []
- for path in docs_to_index:
- resource = root.get_resource(path, soft=True)
- if resource:
- values = resource.get_catalog_values()
- aux.append((resource, values))
- self.backend.flush_catalog(docs_to_unindex, aux)
- self.resources_old2new_catalog.clear()
- self.resources_new2old_catalog.clear()
-
+ try:
+ data = self._before_commit()
+ except Exception as e:
+ log.error("Transaction failed", exc_info=True)
+ try:
+ self._abort_changes()
+ except Exception as e:
+ log.error("Aborting failed", exc_info=True)
+ self._cleanup()
+ raise e
+ _, _, _, docs_to_index, docs_to_unindex = data
+ self.backend.flush_catalog(docs_to_unindex, docs_to_index)
def reindex_catalog(self, base_abspath, recursif=True):
"""Reindex the catalog & return nb resources re-indexed
@@ -563,4 +537,4 @@ def make_database(path, size_min, size_max, fields=None, backend=None):
backend_cls = backends_registry[backend]
backend_cls.init_backend(path, fields)
# Ok
- return RWDatabase(path, size_min, size_max)
+ return RWDatabase(path, size_min, size_max, backend=backend)
diff --git a/itools/datatypes/__init__.py b/itools/datatypes/__init__.py
index 490de92ff..669901051 100644
--- a/itools/datatypes/__init__.py
+++ b/itools/datatypes/__init__.py
@@ -17,16 +17,16 @@
# along with this program. If not, see .
# Import from itools
-from base import DataType
-from primitive import Boolean, Decimal, Email, Integer, String, Unicode
-from primitive import Tokens, MultiLinesTokens, Enumerate
-from primitive import PathDataType, URI
-from primitive import QName, XMLAttribute, XMLContent
-from datetime_ import ISOCalendarDate, ISOTime, ISODateTime, HTTPDate
-from languages import LanguageTag
+from .base import DataType
+from .primitive import Boolean, Decimal, Email, Integer, String, Unicode
+from .primitive import Tokens, MultiLinesTokens, Enumerate
+from .primitive import PathDataType, URI
+from .primitive import QName, XMLAttribute, XMLContent
+from .datetime_ import ISOCalendarDate, ISOTime, ISODateTime, HTTPDate
+from .languages import LanguageTag
# Define alias Date, Time and DateTime (use ISO standard)
-from datetime_ import ISOCalendarDate as Date, ISOTime as Time
-from datetime_ import ISODateTime as DateTime
+from .datetime_ import ISOCalendarDate as Date, ISOTime as Time
+from .datetime_ import ISODateTime as DateTime
__all__ = [
diff --git a/itools/datatypes/base.py b/itools/datatypes/base.py
index af049fd34..54a47581f 100644
--- a/itools/datatypes/base.py
+++ b/itools/datatypes/base.py
@@ -16,9 +16,37 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+# Import from std
+import os
+
+# Import from external
+from cryptography.fernet import Fernet
+from cryptography.fernet import InvalidToken
+
# Import from itools
from itools.core import prototype
+# Fernet is an abstraction implementation of symmetric encryption
+# using AES-256 CBC-MODE with a 32 bytes key
+# https://cryptography.io/en/latest/fernet/#fernet-symmetric-encryption
+
+# To generate a 32 bytes keys use the following methods
+# Oneliner CLI : python -c "import base64;import os;print(base64.urlsafe_b64encode(os.urandom(32)))"
+# In code : Fernet.generate_key()
+FERNET_KEY = os.getenv("FERNET_KEY")
+
+if FERNET_KEY:
+ print(
+ "ENV VAR FERNET_KEY FOR FERNET ENCRYPTION KEY IS SET,"
+ " SENSITIVE VALUES WILL BE ENCRYPTED"
+ )
+ fernet = Fernet(FERNET_KEY)
+else:
+ print(
+ "ENV VAR FERNET_KEY FOR FERNET ENCRYPTION KEY IS NOT SET,"
+ " SENSITIVE VALUES WILL NOT BE ENCRYPTED"
+ )
+ fernet = None
class DataType(prototype):
@@ -26,7 +54,7 @@ class DataType(prototype):
# Default value
default = None
multiple = False
-
+ encrypted = False
def get_default(cls):
default = cls.default
@@ -37,21 +65,18 @@ def get_default(cls):
return []
return default
-
@staticmethod
def decode(data):
- """Deserializes the given byte string to a value with a type.
+ """Deserializes the given str data to a value with a type.
"""
raise NotImplementedError
-
@staticmethod
def encode(value):
- """Serializes the given value to a byte string.
+ """Serializes the given value to str.
"""
raise NotImplementedError
-
@staticmethod
def is_valid(value):
"""Checks whether the given value is valid.
@@ -61,7 +86,6 @@ def is_valid(value):
"""
return True
-
@staticmethod
def is_empty(value):
"""Checks whether the given value is empty or not.
@@ -70,3 +94,30 @@ def is_empty(value):
as empty. (NOTE This is used by the multilingual code.)
"""
return value is None
+
+ # Encryption/Decryption functions
+
+ @classmethod
+ def encrypt(cls, value):
+ if not cls.encrypted:
+ return value
+ if not fernet:
+ # Fernet is not correctly set do not try to encrypt
+ return value
+ if type(value) is str:
+ value = value.encode("utf-8")
+ return fernet.encrypt(value).decode("utf-8")
+
+ @classmethod
+ def decrypt(cls, value):
+ if not cls.encrypted:
+ return value
+ if not fernet:
+ # Fernet is not correctly set do not try to decrypt
+ return value
+ try:
+ if type(value) is str:
+ value = value.encode("utf-8")
+ return fernet.decrypt(value).decode("utf-8")
+ except InvalidToken:
+ return value
diff --git a/itools/datatypes/datetime_.py b/itools/datatypes/datetime_.py
index 7c60b54c4..c514738ea 100644
--- a/itools/datatypes/datetime_.py
+++ b/itools/datatypes/datetime_.py
@@ -27,7 +27,7 @@
# Import from itools
from itools.core import fixed_offset
-from base import DataType
+from .base import DataType
###########################################################################
@@ -60,7 +60,6 @@ def decode(data):
tz = fixed_offset(tz/60)
return tz.localize(naive_dt)
-
@staticmethod
def encode(value):
"""Encode a datetime object to RFC 1123 format: ::
@@ -85,7 +84,6 @@ def encode(value):
# XXX Python dates (the datetime.date module) require the month and day,
# they are not able to represent lower precision dates as ISO 8601 does.
# In the long run we will need to replace Python dates by something else.
-
class ISOCalendarDate(DataType):
"""Extended formats (from max. to min. precision): %Y-%m-%d, %Y-%m, %Y
@@ -96,6 +94,8 @@ class ISOCalendarDate(DataType):
@classmethod
def decode(cls, data):
+ if type(data) is bytes:
+ data = data.decode("utf-8")
if not data:
return None
format_date = cls.format_date
@@ -117,7 +117,6 @@ def decode(cls, data):
return date(year, month, day)
-
@classmethod
def encode(cls, value):
# We choose the extended format as the canonical representation
@@ -125,7 +124,6 @@ def encode(cls, value):
return ''
return value.strftime(cls.format_date)
-
@classmethod
def is_valid(cls, value):
# Only consider the value is valid if we are able to encode it.
@@ -147,7 +145,6 @@ class ISOTime(DataType):
Basic formats: %H%M%S%f, %H%M%S, %H%M, %H
"""
-
@staticmethod
def decode(data):
if not data:
@@ -220,7 +217,6 @@ def decode(data):
microsecond = int(data)
return time(hour, minute, second, microsecond, tzinfo=tzinfo)
-
@staticmethod
def encode(value):
# We choose the extended format as the canonical representation
@@ -232,7 +228,6 @@ def encode(value):
return value.strftime(fmt)
-
class ISODateTime(DataType):
cls_date = ISOCalendarDate
@@ -241,7 +236,8 @@ class ISODateTime(DataType):
def decode(self, value):
if not value:
return None
-
+ if type(value) is bytes:
+ value = value.decode("utf-8")
value = value.split('T')
date, time = value[0], value[1:]
@@ -255,7 +251,6 @@ def decode(self, value):
return date
-
def encode(self, value):
if value is None:
return ''
diff --git a/itools/datatypes/languages.py b/itools/datatypes/languages.py
index 93014d6f7..372705e77 100644
--- a/itools/datatypes/languages.py
+++ b/itools/datatypes/languages.py
@@ -16,7 +16,7 @@
# along with this program. If not, see .
# Import from itools
-from base import DataType
+from .base import DataType
class LanguageTag(DataType):
@@ -29,7 +29,6 @@ def decode(value):
else:
return (res[0].lower(), res[1].upper())
-
@staticmethod
def encode(value):
language, locality = value
diff --git a/itools/datatypes/primitive.py b/itools/datatypes/primitive.py
old mode 100644
new mode 100755
index 067332908..1660ee997
--- a/itools/datatypes/primitive.py
+++ b/itools/datatypes/primitive.py
@@ -30,7 +30,7 @@
from itools.uri import Path, get_reference
# Import from here
-from base import DataType
+from .base import DataType
class Integer(DataType):
@@ -39,15 +39,13 @@ class Integer(DataType):
def decode(value):
if value == '':
return None
- return int(value)
-
+ return int(float(value))
@staticmethod
def encode(value):
if value is None:
return ''
- return str(value)
-
+ return str(int(value))
class Decimal(DataType):
@@ -65,41 +63,41 @@ def encode(value):
return str(value)
-
-class Unicode(DataType):
-
- default = u''
-
+class String(DataType):
@staticmethod
def decode(value, encoding='UTF-8'):
- return unicode(value, encoding).strip()
+ if isinstance(value, bytes):
+ value = value.decode(encoding)
+ return value
@staticmethod
def encode(value, encoding='UTF-8'):
- return value.strip().encode(encoding)
-
+ if value is None:
+ return ""
+ if isinstance(value, bytes):
+ value = value.decode(encoding)
+ return value
@staticmethod
def is_empty(value):
- return value == u''
-
+ return value == ''
-class String(DataType):
-
- @staticmethod
- def decode(value):
- return value
+class Unicode(String):
+ """
+ This exists only for backwards compatibility, to make migration to Pyhon 3
+ easier.
+ The only difference with String is the default value (empty string), and
+ that itools.catalog will split Unicode in words when indexing.
- @staticmethod
- def encode(value):
- if value is None:
- return ''
- return value
+ Text would be a better name than Unicode, but we keep Unicode so we don't
+ have to change too much code.
+ """
+ default = ''
class Boolean(DataType):
@@ -111,7 +109,6 @@ class Boolean(DataType):
def decode(value):
return bool(int(value))
-
@staticmethod
def encode(value):
if value is True:
@@ -122,7 +119,6 @@ def encode(value):
raise ValueError('{0} value is not a boolean'.format(value))
-
class URI(String):
# XXX Should we at least normalize the sring when decoding/encoding?
@@ -134,13 +130,11 @@ def is_valid(value):
return False
return True
-
@staticmethod
def is_empty(value):
return not value
-
class PathDataType(DataType):
# TODO Do like URI, do not decode (just an string), and use 'is_valid'
# instead
@@ -157,7 +151,6 @@ def encode(value):
return str(value)
-
# We consider the local part in emails is case-insensitive. This is against
# the standard, but corresponds to common usage.
email_expr = "^[0-9a-z]+[_\.0-9a-z-'+]*@([0-9a-z-]+\.)+[a-z]{2,6}$"
@@ -170,6 +163,8 @@ def encode(value):
@staticmethod
def decode(value):
+ if isinstance(value, bytes):
+ value = value.decode("utf-8")
return value.lower()
@staticmethod
@@ -177,7 +172,6 @@ def is_valid(value):
return email_expr.match(value) is not None
-
class QName(DataType):
@staticmethod
@@ -187,7 +181,6 @@ def decode(data):
return None, data
-
@staticmethod
def encode(value):
if value[0] is None:
@@ -195,7 +188,6 @@ def encode(value):
return '%s:%s' % value
-
class Tokens(DataType):
default = ()
@@ -204,36 +196,29 @@ class Tokens(DataType):
def decode(data):
return tuple(data.split())
-
@staticmethod
def encode(value):
return ' '.join(value)
-
class MultiLinesTokens(DataType):
@staticmethod
def decode(data):
return tuple(data.splitlines())
-
@staticmethod
def encode(value):
return '\n'.join(value)
-
-
###########################################################################
# Enumerates
-
class Enumerate(String):
is_enumerate = True
options = freeze([])
-
def get_options(cls):
"""Returns a list of dictionaries in the format
[{'name': , 'value': }, ...]
@@ -243,7 +228,6 @@ def get_options(cls):
"""
return deepcopy(cls.options)
-
def is_valid(self, name):
"""Returns True if the given name is part of this Enumerate's options.
"""
@@ -256,7 +240,6 @@ def is_valid(self, name):
return True
return False
-
def get_namespace(cls, name):
"""Extends the options with information about which one is matching
the given name.
@@ -264,7 +247,6 @@ def get_namespace(cls, name):
options = cls.get_options()
return enumerate_get_namespace(options, name)
-
def get_value(cls, name, default=None):
"""Returns the value matching the given name, or the default value.
"""
@@ -298,19 +280,21 @@ class XMLContent(object):
@staticmethod
def encode(value):
+ if isinstance(value, bytes):
+ value = value.decode("utf-8")
return value.replace('&', '&').replace('<', '<')
-
@staticmethod
def decode(value):
return value.replace('&', '&').replace('<', '<')
-
class XMLAttribute(object):
@staticmethod
def encode(value):
+ if isinstance(value, bytes):
+ value = value.decode("utf-8")
value = value.replace('&', '&').replace('<', '<')
return value.replace('"', '"')
@@ -349,7 +333,6 @@ def encode(value):
return dumps(value, cls=NewJSONEncoder)
-
class JSONArray(JSONObject):
"""A JSON array, which is a Python list serialized as a JSON string
@@ -358,7 +341,6 @@ class JSONArray(JSONObject):
default = []
-
@staticmethod
def is_valid(value):
return isinstance(value, list)
diff --git a/itools/fs/__init__.py b/itools/fs/__init__.py
index 1cc0dad40..14d720ab3 100644
--- a/itools/fs/__init__.py
+++ b/itools/fs/__init__.py
@@ -18,8 +18,8 @@
# along with this program. If not, see .
# Import from itools
-from common import READ, WRITE, READ_WRITE, APPEND, FileName
-from lfs import lfs
+from .common import READ, WRITE, READ_WRITE, APPEND, FileName
+from .lfs import lfs
__all__ = [
diff --git a/itools/fs/common.py b/itools/fs/common.py
index 1618e607a..bacb31f12 100644
--- a/itools/fs/common.py
+++ b/itools/fs/common.py
@@ -24,9 +24,8 @@
READ = 'r'
WRITE = 'w'
-READ_WRITE = 'rw'
-APPEND = 'a'
-
+READ_WRITE = 'w+'
+APPEND = 'a+'
class FileName(DataType):
@@ -73,8 +72,6 @@ def encode(value):
return name
-
-
def get_mimetype(name):
"""Try to guess the mimetype given the name. To guess from the name we
need to extract the type extension, we use an heuristic for this task,
diff --git a/itools/fs/lfs.py b/itools/fs/lfs.py
old mode 100644
new mode 100755
index 3a2d939b8..db9aa57b2
--- a/itools/fs/lfs.py
+++ b/itools/fs/lfs.py
@@ -22,27 +22,33 @@
from datetime import datetime
from os import listdir, makedirs, remove as os_remove, renames, walk
from os import access, R_OK, W_OK
-from os.path import exists, getatime, getctime, getmtime ,getsize
+from os.path import exists, getatime, getctime, getmtime, getsize
from os.path import isfile, isdir, join, basename, dirname
from os.path import abspath, relpath
from shutil import rmtree, copytree, copy as shutil_copy
+import mimetypes
+
# Import from itools
from itools.uri import Path
-from common import WRITE, READ_WRITE, APPEND, READ, get_mimetype
-
+from .common import WRITE, READ_WRITE, APPEND, READ, get_mimetype
-MODES = {WRITE: 'wb', READ_WRITE: 'r+b', APPEND: 'ab', READ: 'rb'}
+MODES = {
+ WRITE: ('w', 'wb'),
+ READ_WRITE: ('w+', 'wb+'),
+ APPEND: ('a+', 'ab+'),
+ READ: ('r', 'rb'),
+}
class LocalFolder(object):
def __init__(self, path='.'):
if not exists(path):
- raise IOError, "No such directory: '%s'" % path
+ raise IOError("No such directory: '%s'" % path)
if isfile(path):
- raise IOError, "Is a directory: '%s'" % path
+ raise IOError("Is a directory: '%s'" % path)
self.path = Path(abspath(path))
@@ -78,23 +84,23 @@ def can_write(self, path):
path = self._resolve_path(path)
return access(path, W_OK)
-
- def make_file(self, path):
+ def make_file(self, path, text=False):
path = self._resolve_path(path)
parent_path = dirname(path)
if exists(parent_path):
if exists(path):
- raise OSError, "File exists: '%s'" % path
+ raise OSError("File exists: '%s'" % path)
else:
makedirs(parent_path)
- return file(path, 'wb')
-
+ if text:
+ return open(path, 'w')
+ else:
+ return open(path, 'wb')
def make_folder(self, path):
path = self._resolve_path(path)
return makedirs(path)
-
def get_ctime(self, path):
path = self._resolve_path(path)
ctime = getctime(path)
@@ -126,14 +132,16 @@ def get_size(self, path):
path = self._resolve_path(path)
return getsize(path)
-
- def open(self, path, mode=None):
+ def open(self, path, mode=None, text=False):
path = self._resolve_path(path)
if isdir(path):
return self.__class__(path)
- mode = MODES.get(mode, 'rb')
- return file(path, mode)
-
+ mode = MODES.get(mode, ('r', 'rb'))
+ if text:
+ mode = mode[0]
+ else:
+ mode = mode[1]
+ return open(path, mode)
def remove(self, path):
path = self._resolve_path(path)
@@ -141,7 +149,11 @@ def remove(self, path):
# Remove folder contents
rmtree(path)
else:
- os_remove(path)
+ try:
+ os_remove(path)
+ except OSError as error:
+ print(error)
+ print("File path can not be removed")
def copy(self, source, target):
@@ -167,7 +179,7 @@ def get_names(self, path='.'):
path = self._resolve_path(path)
try:
return listdir(path)
- except OSError, e:
+ except OSError as e:
# Path does not exist or is not a directory
if e.errno == 2 or e.errno == 20:
return []
@@ -177,7 +189,7 @@ def get_names(self, path='.'):
def traverse(self, path='.'):
path = self._resolve_path(path)
if not exists(path):
- raise IOError, "No such directory: '%s'" % path
+ raise IOError("No such directory: '%s'" % path)
yield path
if isdir(path):
for root, folders, files in walk(path, topdown=True):
diff --git a/itools/gettext/__init__.py b/itools/gettext/__init__.py
index 5523805bd..4e872782d 100644
--- a/itools/gettext/__init__.py
+++ b/itools/gettext/__init__.py
@@ -18,9 +18,9 @@
# Import from itools
from itools.core import add_type, get_abspath
-from domains import register_domain, get_domain, MSG, get_language_msg
-from mo import MOFile
-from po import POFile, POUnit, encode_source
+from .domains import register_domain, get_domain, MSG, get_language_msg
+from .mo import MOFile
+from .po import POFile, POUnit, encode_source
__all__ = [
diff --git a/itools/gettext/domains.py b/itools/gettext/domains.py
index 0c09a2dcd..a7a81e48c 100644
--- a/itools/gettext/domains.py
+++ b/itools/gettext/domains.py
@@ -22,13 +22,12 @@
from sys import _getframe
# Import from itools
-from itools.handlers import Folder
from itools.i18n import get_language_name
from itools.fs import lfs
from itools.xml import XMLParser
# Import from here
-from mo import MOFile
+from .mo import MOFile
xhtml_namespaces = {
@@ -41,37 +40,40 @@
domains = {}
+
def register_domain(name, locale_path):
if name not in domains:
- domains[name] = Domain(locale_path)
+ domains[name] = locale_path
def get_domain(name):
- return domains[name]
-
+ domain = domains.get(name)
+ # Lazy load domain
+ if isinstance(domain, str):
+ domain = Domain(domain)
+ domains[name] = domain
+ return domain
class Domain(dict):
def __init__(self, uri):
+ super().__init__()
for key in lfs.get_names(uri):
if key[-3:] == '.mo':
language = key[:-3]
path = '{0}/{1}'.format(uri, key)
self[language] = MOFile(path)
-
def gettext(self, message, language):
if language not in self:
return message
handler = self[language]
return handler.gettext(message)
-
def get_languages(self):
- return self.keys()
-
+ return list(self.keys())
class MSGFormatter(Formatter):
@@ -94,6 +96,7 @@ def get_value(self, key, args, kw):
msg_formatter = MSGFormatter()
+
class MSG(object):
domain = None
@@ -109,26 +112,22 @@ def __init__(self, message=None, format=None):
if message:
self.message = message
-
def _format(self, message, **kw):
if self.format == 'replace':
return msg_formatter.vformat(message, [], (self, kw))
elif self.format == 'none':
return message
elif self.format == 'html':
- data = message.encode('utf_8')
- return XMLParser(data, namespaces=xhtml_namespaces)
+ return XMLParser(message, namespaces=xhtml_namespaces)
elif self.format == 'replace_html':
message = msg_formatter.vformat(message, [], (self, kw))
- data = message.encode('utf_8')
- return XMLParser(data, namespaces=xhtml_namespaces)
-
- raise ValueError, 'unexpected format "{0}"'.format(self.format)
+ return XMLParser(message, namespaces=xhtml_namespaces)
+ raise ValueError('unexpected format "{0}"'.format(self.format))
def gettext(self, language=None, **kw):
message = self.message
- domain = domains.get(self.domain)
+ domain = get_domain(self.domain)
if domain is not None:
# Find out the language
@@ -144,7 +143,6 @@ def gettext(self, language=None, **kw):
return self._format(message, **kw)
-
def get_language_msg(code):
language = get_language_name(code)
return MSG(language)
diff --git a/itools/gettext/mo.py b/itools/gettext/mo.py
index f5c9ce18c..00ebf6a3a 100644
--- a/itools/gettext/mo.py
+++ b/itools/gettext/mo.py
@@ -30,11 +30,10 @@ class MOFile(File):
def _load_state_from_file(self, file):
self.translations = GNUTranslations(file)
-
def gettext(self, message):
"""Returns the translation for the given message.
"""
- return self.translations.ugettext(message)
+ return self.translations.gettext(message)
register_handler_class(MOFile)
diff --git a/itools/gettext/po.py b/itools/gettext/po.py
index 7789c434a..ddc51acea 100644
--- a/itools/gettext/po.py
+++ b/itools/gettext/po.py
@@ -46,7 +46,6 @@ def __init__(self, line_number, line_type=None):
self.line_number = line_number
self.line_type = line_type
-
def __str__(self):
if self.line_type is None:
return 'syntax error at line %d' % self.line_number
@@ -77,6 +76,9 @@ def __str__(self):
def get_lines(data):
+ if isinstance(data, bytes):
+ data = data.decode("utf-8")
+
lines = data.split('\n') + ['']
line_number = 0
while line_number <= len(lines):
@@ -97,7 +99,7 @@ def get_lines(data):
yield FUZZY, None, line_number
# Reference
elif line.startswith('#:'):
- yield REFERENCE, line[1:], line_number
+ yield REFERENCE, line[2:].strip(), line_number
# Comment
elif line.startswith('#'):
yield COMMENT, line[1:], line_number
@@ -139,14 +141,14 @@ def encode_source(source):
elif type == START_FORMAT:
# A lonely tag ?
if source[i+1][0] == END_FORMAT:
- result.append(u"" % value)
+ result.append("" % value)
else:
- result.append(u"" % value)
+ result.append("" % value)
elif type == END_FORMAT:
# A lonely tag ?
if source[i-1][0] != START_FORMAT:
- result.append(u'')
- return u''.join(result)
+ result.append('')
+ return ''.join(result)
def decode_target(target):
@@ -202,7 +204,6 @@ def decode_target(target):
return result
-
###########################################################################
# Handler
###########################################################################
@@ -213,11 +214,12 @@ def escape(s):
expr = compile(r'(\\.)')
+
+
def unescape(s):
return expr.sub(lambda x: eval("'%s'" % x.group(0)), s)
-
class POUnit(object):
"""An entry in a PO file has the syntax:
@@ -259,12 +261,11 @@ def __init__(self, comments, context, source, target,
self.target = target
self.fuzzy = fuzzy
-
def to_str(self, encoding='UTF-8'):
s = []
# The comments
for comment in self.comments:
- s.append('#%s\n' % comment.encode(encoding))
+ s.append('#%s\n' % comment)
# The reference comments
i = 1
references = self.references.items()
@@ -274,40 +275,35 @@ def to_str(self, encoding='UTF-8'):
comma = '' if i == nb_references else ','
line = '#: %s:%s%s\n' % (filename, line, comma)
s.append(line)
- i+=1
+ i += 1
# The Fuzzy flag
if self.fuzzy:
s.append('#, fuzzy\n')
# The msgctxt
if self.context is not None:
- s.append('msgctxt "%s"\n' % escape(self.context[0].encode(
- encoding)))
+ s.append('msgctxt "%s"\n' % escape(self.context[0]))
for string in self.context[1:]:
- s.append('"%s"\n' % escape(string.encode(encoding)))
+ s.append('"%s"\n' % escape(string))
# The msgid
- s.append('msgid "%s"\n' % escape(self.source[0].encode(encoding)))
+ s.append('msgid "%s"\n' % escape(self.source[0]))
for string in self.source[1:]:
- s.append('"%s"\n' % escape(string.encode(encoding)))
+ s.append('"%s"\n' % escape(string))
# The msgstr
- s.append('msgstr "%s"\n' % escape(self.target[0].encode(encoding)))
+ s.append('msgstr "%s"\n' % escape(self.target[0]))
for string in self.target[1:]:
- s.append('"%s"\n' % escape(string.encode(encoding)))
-
+ s.append('"%s"\n' % escape(string))
return ''.join(s)
-
def __repr__(self):
msg = ""
return msg % (self.context, self.source, self.target, self.references)
-
def __eq__(self, other):
return ((other.context == self.context) and
(other.source == self.source) and
(other.target == self.target))
-
skeleton = """# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
@@ -327,7 +323,6 @@ def __eq__(self, other):
"""
-
class POFile(TextFile):
class_mimetypes = [
@@ -336,7 +331,6 @@ class POFile(TextFile):
'text/x-po']
class_extension = 'po'
-
def new(self):
# XXX Old style (like in the "get_skeleton" times)
now = strftime('%Y-%m-%d %H:%m+%Z', gmtime(time()))
@@ -366,7 +360,7 @@ def next_entry(self, data):
comments.append(value)
state = 1
elif line_type == REFERENCE:
- for reference in value[3:].split(' '):
+ for reference in value.split(' '):
value, line_no = reference.split(':')
references.append((value, line_no))
state = 1
@@ -387,7 +381,7 @@ def next_entry(self, data):
if line_type == COMMENT:
comments.append(value)
elif line_type == REFERENCE:
- for reference in value[3:].split(' '):
+ for reference in value.split(' '):
value, line_no = reference.split(':')
references.append((value, line_no))
elif line_type == FUZZY and not fuzzy:
@@ -479,7 +473,6 @@ def next_entry(self, data):
else:
raise POSyntaxError(line_number, line_type)
-
def _load_state_from_file(self, file):
"""A PO file is made of entries, where entries are separated by one
or more blank lines. Each entry consists of a msgid and a msgstr,
@@ -504,7 +497,6 @@ def _load_state_from_file(self, file):
# Split the data by lines and intialize the line index
data = file.read()
-
# Add entries
for entry in self.next_entry(data):
comments, references, context, source, target, fuzzy, line_number = entry
@@ -517,39 +509,37 @@ def _load_state_from_file(self, file):
second_part = ''.join(source)
key = (first_part, second_part)
if key in self.messages:
- raise POError, 'msgid at line %d is duplicated' % line_number
+ raise POError('msgid at line %d is duplicated' % line_number)
# Get the comments and the msgstr in unicode
- comments = [ unicode(x, self.encoding) for x in comments ]
-
- if context is not None:
- context = [ unicode(x, self.encoding) for x in context ]
- source = [ unicode(x, self.encoding) for x in source ]
- target = [ unicode(x, self.encoding) for x in target ]
-
+ # No need to encoding Python 3
+ # comments = [str(x, self.encoding) for x in comments]
+ #
+ # if context is not None:
+ # context = [str(x, self.encoding) for x in context]
+ # source = [str(x, self.encoding) for x in source]
+ # target = [str(x, self.encoding) for x in target]
# Add the message
self._set_message(context, source, target, comments, references, fuzzy)
-
def to_str(self, encoding='UTF-8'):
messages = self.messages
- message_ids = messages.keys()
- message_ids.sort()
- messages = [ messages[x].to_str(encoding) for x in message_ids ]
+ message_ids = sorted(list(messages.keys()), key=lambda x: x[1])
+ messages = [messages[x].to_str(encoding) for x in message_ids]
return '\n'.join(messages)
#######################################################################
# API / Private
#######################################################################
- def _set_message(self, context, source, target=freeze([u'']),
+ def _set_message(self, context, source, target=freeze(['']),
comments=freeze([]), references=None, fuzzy=False):
- if context is not None and isinstance(context, (str, unicode)):
+ if context is not None and isinstance(context, (str, str)):
context = [context]
- if isinstance(source, (str, unicode)):
+ if isinstance(source, (str, str)):
source = [source]
- if isinstance(target, (str, unicode)):
+ if isinstance(target, (str, str)):
target = [target]
# Make the key
@@ -573,25 +563,21 @@ def _set_message(self, context, source, target=freeze([u'']),
if not references:
return unit
for reference in references:
- unit.references.setdefault(str(reference[0]), []).append(reference[1])
+ unit.references.setdefault(reference[0], []).append(reference[1])
return unit
-
-
#######################################################################
# API / Public
#######################################################################
def get_msgids(self):
"""Returns all the (context, msgid).
"""
- return self.messages.keys()
-
+ return list(self.messages.keys())
def get_units(self):
"""Returns all the message (objects of the class ).
"""
- return self.messages.values()
-
+ return list(self.messages.values())
def get_msgstr(self, source, context=None):
"""Returns the 'msgstr' for the given (context, msgid).
@@ -601,11 +587,9 @@ def get_msgstr(self, source, context=None):
return ''.join(message.target)
return None
-
def set_msgstr(self, source, target, context=None):
self._set_message(context, [source], [target])
-
def gettext(self, source, context=None):
"""Returns the translation of the given message id.
@@ -620,7 +604,6 @@ def gettext(self, source, context=None):
return decode_target(target)
return source
-
def add_unit(self, filename, source, context, line):
if not source:
return None
@@ -630,9 +613,8 @@ def add_unit(self, filename, source, context, line):
source = encode_source(source)
- return self._set_message(context, [source], [u''], [],
+ return self._set_message(context, [source], [''], [],
[(filename, line)])
-
register_handler_class(POFile)
diff --git a/itools/handlers/__init__.py b/itools/handlers/__init__.py
index 60236ae9c..b76f4da39 100644
--- a/itools/handlers/__init__.py
+++ b/itools/handlers/__init__.py
@@ -31,16 +31,16 @@
"""
# Import from itools
-from archive import ZIPFile, TARFile, GzipFile, Bzip2File, TGZFile, TBZ2File
-from base import Handler
-from config import ConfigFile
-from file import File
-from folder import Folder
-from image import Image, SVGFile
-from js import JSFile
-from registry import register_handler_class, get_handler_class_by_mimetype
-from text import TextFile, guess_encoding
-from utils import checkid
+from .archive import ZIPFile, TARFile, GzipFile, Bzip2File, TGZFile, TBZ2File
+from .base import Handler
+from .config import ConfigFile
+from .file import File
+from .folder import Folder
+from .image import Image, SVGFile
+from .js import JSFile
+from .registry import register_handler_class, get_handler_class_by_mimetype
+from .text import TextFile, guess_encoding
+from .utils import checkid
__all__ = [
diff --git a/itools/handlers/archive.py b/itools/handlers/archive.py
old mode 100644
new mode 100755
index ef15465a6..258bef493
--- a/itools/handlers/archive.py
+++ b/itools/handlers/archive.py
@@ -21,12 +21,11 @@
from os.path import join
from zipfile import ZipFile
from tarfile import open as open_tarfile
-from cStringIO import StringIO
+from io import StringIO, BytesIO
# Import from itools
-from file import File
-from registry import register_handler_class
-
+from .file import File
+from .registry import register_handler_class
class Info(object):
@@ -40,18 +39,21 @@ def __init__(self, name, mtime):
self.mtime = mtime
-
class ZIPFile(File):
class_mimetypes = ['application/zip']
class_extension = 'zip'
-
def _open_zipfile(self):
- archive = StringIO(self.to_str())
+ data = self.to_str()
+ if isinstance(data, bytes):
+ archive = BytesIO(data)
+ elif isinstance(data, str):
+ archive = StringIO(data)
+ else:
+ raise Exception("Error Zipfile")
return ZipFile(archive)
-
def get_members(self):
zip = self._open_zipfile()
try:
@@ -64,7 +66,6 @@ def get_members(self):
finally:
zip.close()
-
def get_contents(self):
zip = self._open_zipfile()
try:
@@ -72,7 +73,6 @@ def get_contents(self):
finally:
zip.close()
-
def get_file(self, filename):
zip = self._open_zipfile()
try:
@@ -80,7 +80,6 @@ def get_file(self, filename):
finally:
zip.close()
-
def extract_to_folder(self, dst):
zip = self._open_zipfile()
try:
@@ -93,19 +92,16 @@ def extract_to_folder(self, dst):
zip.close()
-
class TARFile(File):
class_mimetypes = ['application/x-tar']
class_extension = 'tar'
class_mode = 'r'
-
def _open_tarfile(self):
archive = StringIO(self.to_str())
return open_tarfile(mode=self.class_mode, fileobj=archive)
-
def get_members(self):
tar = self._open_tarfile()
try:
@@ -120,7 +116,6 @@ def get_members(self):
finally:
tar.close()
-
def get_contents(self):
tar = self._open_tarfile()
try:
@@ -131,7 +126,6 @@ def get_contents(self):
finally:
tar.close()
-
def get_file(self, filename):
tar = self._open_tarfile()
try:
@@ -139,7 +133,6 @@ def get_file(self, filename):
finally:
tar.close()
-
def extract_to_folder(self, dst):
tar = self._open_tarfile()
try:
@@ -148,7 +141,6 @@ def extract_to_folder(self, dst):
tar.close()
-
class TGZFile(TARFile):
class_mimetypes = ['application/x-tgz']
@@ -156,7 +148,6 @@ class TGZFile(TARFile):
class_mode = 'r:gz'
-
class TBZ2File(TARFile):
class_mimetypes = ['application/x-tbz2']
@@ -164,21 +155,18 @@ class TBZ2File(TARFile):
class_mode = 'r:bz2'
-
class GzipFile(File):
class_mimetypes = ['application/x-gzip']
class_extension = 'gz'
-
class Bzip2File(File):
class_mimetypes = ['application/x-bzip2']
class_extension = 'bz2'
-
# Register
for cls in [ZIPFile, TARFile, TGZFile, TBZ2File, GzipFile, Bzip2File]:
register_handler_class(cls)
diff --git a/itools/handlers/base.py b/itools/handlers/base.py
index 9b9c32bec..17152198f 100644
--- a/itools/handlers/base.py
+++ b/itools/handlers/base.py
@@ -37,80 +37,71 @@ class Handler(object):
database = None
key = None
-
def has_handler(self, reference):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
return database.has_handler(key)
-
def get_handler_names(self, reference='.'):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
return database.get_handler_names(key)
-
def get_handler(self, reference, cls=None, soft=False):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
return database._get_handler(key, cls, soft)
-
def get_handlers(self, reference='.'):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
return database.get_handlers(key)
-
def set_handler(self, reference, handler):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
database.set_handler(key, handler)
-
def del_handler(self, reference):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
key = database.resolve2(self.key, reference)
database.del_handler(key)
-
def copy_handler(self, source, target, exclude_patterns=None):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
source = database.resolve2(self.key, source)
target = database.resolve2(self.key, target)
database.copy_handler(source, target, exclude_patterns)
-
def move_handler(self, source, target):
database = self.database
if database is None:
- raise NotImplementedError, MSG_NOT_ATTACHED
+ raise NotImplementedError(MSG_NOT_ATTACHED)
source = database.resolve2(self.key, source)
target = database.resolve2(self.key, target)
database.move_handler(source, target)
-
def get_mimetype(self):
return self.database.get_mimetype(self.key)
diff --git a/itools/handlers/config.py b/itools/handlers/config.py
index fc7c85618..690e667f3 100644
--- a/itools/handlers/config.py
+++ b/itools/handlers/config.py
@@ -27,13 +27,14 @@
# Import from itools
from itools.datatypes import String
-from text import TextFile
+from .text import TextFile
###########################################################################
# Lines Analyser (an automaton)
###########################################################################
-BLANK, COMMENT, VAR, VAR_START, VAR_CONT, VAR_END, EOF = range(7)
+BLANK, COMMENT, VAR, VAR_START, VAR_CONT, VAR_END, EOF = list(range(7))
+
def get_lines(file):
"""Analyses the physical lines, identifies the type and parses them.
@@ -89,15 +90,15 @@ def get_lines(file):
value = ''.join(groups)
value = value.replace('\\"','"')
if nb_groups > 3:
- raise SyntaxError, 'unescaped char, line %d' % line_num
+ raise SyntaxError('unescaped char, line %d' % line_num)
- if nb_groups in (3,1):
+ if nb_groups in (3, 1):
yield VAR, (name, value), line_num
if nb_groups == 2:
yield VAR_START, (name, value), line_num
state = 1
else:
- raise SyntaxError, 'unknown line "%d"' % line_num
+ raise SyntaxError('unknown line "%d"' % line_num)
elif state == 1:
# Multiline value
if line == '"':
@@ -106,9 +107,9 @@ def get_lines(file):
groups = split('(?<=[^\\\\])"', line)
nb_groups = len(groups)
value = groups[0]
- value = value.replace('\\"','"')
+ value = value.replace('\\"', '"')
if nb_groups > 2:
- raise SyntaxError, 'unescaped char, line %d' % line_num
+ raise SyntaxError('unescaped char, line %d' % line_num)
elif nb_groups == 2:
yield VAR_END, value, line_num
state = 0
@@ -126,11 +127,11 @@ class Lines(object):
def __init__(self, file):
self.lines = get_lines(file)
- self.next()
+ next(self)
- def next(self):
- self.current = self.lines.next()
+ def __next__(self):
+ self.current = next(self.lines)
@@ -168,26 +169,26 @@ def read_block(lines):
"""
type, value, line_num = lines.current
if type == BLANK:
- lines.next()
+ next(lines)
return None
elif type == COMMENT:
- lines.next()
+ next(lines)
comment = [value] + read_comment(lines)
variable = read_variable(lines)
return comment, variable
elif type == VAR:
- lines.next()
+ next(lines)
return [], value
elif type == VAR_START:
name, value = value
- lines.next()
+ next(lines)
value = value + '\n' + read_multiline(lines)
return [], (name, value)
elif type == VAR_CONT:
- lines.next()
+ next(lines)
return None
else:
- raise SyntaxError, 'unexpected line "%d"' % line_num
+ raise SyntaxError('unexpected line "%d"' % line_num)
def read_comment(lines):
@@ -198,7 +199,7 @@ def read_comment(lines):
"""
type, value, line_num = lines.current
if type == COMMENT:
- lines.next()
+ next(lines)
return [value] + read_comment(lines)
return []
@@ -211,11 +212,11 @@ def read_variable(lines):
"""
type, value, line_num = lines.current
if type == VAR:
- lines.next()
+ next(lines)
return value
elif type == VAR_START:
name, value = value
- lines.next()
+ next(lines)
return name, value + '\n' + read_multiline(lines)
@@ -227,13 +228,13 @@ def read_multiline(lines):
"""
type, value, line_num = lines.current
if type == VAR_CONT:
- lines.next()
+ next(lines)
return value + '\n' + read_multiline(lines)
elif type == VAR_END:
- lines.next()
+ next(lines)
return value
else:
- raise SyntaxError, 'unexpected line "%s"' % line_num
+ raise SyntaxError('unexpected line "%s"' % line_num)
###########################################################################
@@ -272,7 +273,6 @@ class ConfigFile(TextFile):
class_extension = None
schema = None
-
def new(self, **kw):
# Comments are not supported here
self.values = {}
@@ -281,7 +281,7 @@ def new(self, **kw):
n = 0
for name, value in kw.items():
if isinstance(value, str) is False:
- raise TypeError, 'the value must be a string.'
+ raise TypeError('the value must be a string.')
# Add the variable, with an empty comment
self.lines.append(([], (name, value)))
# Separate with a blank line
@@ -291,14 +291,13 @@ def new(self, **kw):
# Next
n += 2
-
def _load_state_from_file(self, file):
self.lines, self.values = parse(file)
-
def to_str(self):
lines = []
for line in self.lines:
+
if line is None:
# Blank line
lines.append('\n')
@@ -313,7 +312,6 @@ def to_str(self):
return ''.join(lines)
-
#########################################################################
# API
#########################################################################
@@ -328,7 +326,7 @@ def set_value(self, name, value, comment=None):
if self.schema is not None and name in self.schema:
value = self.schema[name].encode(value)
if not isinstance(value, str):
- raise TypeError, 'the value must be a string.'
+ raise TypeError('the value must be a string.')
self.set_changed()
if name in self.values:
@@ -350,7 +348,6 @@ def set_value(self, name, value, comment=None):
# Append a blank line
self.lines.append(None)
-
def append_comment(self, comment):
"""
Appends a solitary comment.
@@ -364,7 +361,6 @@ def append_comment(self, comment):
# Append a blank line
self.lines.append(None)
-
def get_value(self, name, type=None, default=None):
if name not in self.values:
if default is not None:
@@ -392,7 +388,6 @@ def get_value(self, name, type=None, default=None):
return type.decode(value)
-
def get_comment(self, name):
if name not in self.values:
return None
@@ -402,6 +397,5 @@ def get_comment(self, name):
# Return the comment
return ' '.join(line[0])
-
def has_value(self, name):
return name in self.values
diff --git a/itools/handlers/file.py b/itools/handlers/file.py
old mode 100644
new mode 100755
index 8a2592b87..af5547987
--- a/itools/handlers/file.py
+++ b/itools/handlers/file.py
@@ -18,15 +18,17 @@
# along with this program. If not, see .
# Import from the Standard Library
+from logging import getLogger
from copy import deepcopy
-from cStringIO import StringIO
+from io import StringIO, BytesIO
from datetime import datetime
-from sys import exc_info
+
# Import from itools.handlers
-from base import Handler
-from registry import register_handler_class
+from .base import Handler
+from .registry import register_handler_class
+log = getLogger("itools.database")
class File(Handler):
@@ -49,26 +51,31 @@ class File(Handler):
"""
class_mimetypes = ['application/octet-stream']
+ is_text = False
# By default handlers are not loaded
timestamp = None
dirty = None
loaded = False
-
def __init__(self, key=None, string=None, database=None, **kw):
if database is not None:
self.database = database
else:
+
try:
from itools.database.ro import ro_database
self.database = ro_database
- except:
+ except Exception as e:
+ print(e)
if key:
- print('Cannot attach handler {0} to a database'.format(key))
- with open(key, 'r') as f:
- string = f.read()
+ log.warning('Cannot attach handler {0} to a database'.format(key))
+ try:
+ string = open(key, 'r').read()
+ except UnicodeDecodeError:
+ string = open(key, 'rb').read()
key = None
+
if key is None:
self.reset()
self.dirty = datetime.now()
@@ -82,11 +89,9 @@ def __init__(self, key=None, string=None, database=None, **kw):
self.key = self.database.normalize_key(key)
self.load_state()
-
def reset(self):
pass
-
def new(self, data=''):
self.data = data
@@ -97,21 +102,19 @@ def _load_state_from_file(self, file):
"""Method to be overriden by sub-classes."""
self.data = file.read()
-
def load_state(self):
- data = self.database.get_handler_data(self.key)
+ data = self.database.get_handler_data(self.key, text=self.is_text)
self.reset()
try:
self.load_state_from_string(data)
except Exception as e:
# Update message to add the problematic file
- message = '{0} on "{1}"'.format(e.message, self.key)
+ message = '{0} on "{1}"'.format(e, self.key)
self._clean_state()
raise
self.timestamp = self.database.get_handler_mtime(self.key)
self.dirty = None
-
def load_state_from_file(self, file):
self.reset()
try:
@@ -121,12 +124,15 @@ def load_state_from_file(self, file):
raise
self.loaded = True
-
def load_state_from_string(self, string):
- file = StringIO(string)
+ if isinstance(string, bytes):
+ file = BytesIO(string)
+ elif isinstance(string, str):
+ file = StringIO(string)
+ else:
+ raise Exception(f"String type error {type(string)}")
self.load_state_from_file(file)
-
def save_state(self):
if not self.dirty:
return
@@ -136,19 +142,17 @@ def save_state(self):
self.timestamp = self.database.get_handler_mtime(self.key)
self.dirty = None
-
def save_state_to(self, key):
self.database.save_handler(key, self)
-
clone_exclude = frozenset(['database', 'key', 'timestamp', 'dirty'])
+
def clone(self, cls=None):
# Define the class to build
if cls is None:
cls = self.__class__
elif not issubclass(cls, self.__class__):
- msg = 'the given class must be a subclass of the object'
- raise ValueError, msg
+ raise ValueError("the given class must be a subclass of the object")
# Load first, if needed
if self.dirty is None:
@@ -166,13 +170,12 @@ def clone(self, cls=None):
copy.dirty = datetime.now()
return copy
-
def set_changed(self):
# Set as changed
key = self.key
# Invalid handler
if key is None and self.dirty is None:
- raise RuntimeError, 'cannot change an orphaned file handler'
+ raise RuntimeError('cannot change an orphaned file handler')
# Set as dirty
self.dirty = datetime.now()
# Free handler (not attached to a database)
@@ -182,13 +185,11 @@ def set_changed(self):
# Attached
database.touch_handler(key, self)
-
def _clean_state(self):
- names = [ x for x in self.__dict__ if x not in ('database', 'key') ]
+ names = [x for x in self.__dict__ if x not in ('database', 'key')]
for name in names:
delattr(self, name)
-
def abort_changes(self):
# Not attached to a key or not changed
if self.key is None or self.dirty is None:
@@ -198,7 +199,6 @@ def abort_changes(self):
# Reload state
self.load_state()
-
#########################################################################
# API
#########################################################################
@@ -216,23 +216,18 @@ def get_mtime(self):
# Not yet loaded, check the FS
return self.database.get_handler_mtime(self.key)
-
def to_str(self):
return self.data
-
def set_data(self, data):
self.set_changed()
self.data = data
-
def to_text(self):
raise NotImplementedError
-
def is_empty(self):
raise NotImplementedError
-
register_handler_class(File)
diff --git a/itools/handlers/folder.py b/itools/handlers/folder.py
index bd1647d77..4ade15c23 100644
--- a/itools/handlers/folder.py
+++ b/itools/handlers/folder.py
@@ -17,9 +17,8 @@
# along with this program. If not, see .
# Import from itools
-from base import Handler
-from registry import register_handler_class
-
+from .base import Handler
+from .registry import register_handler_class
class Context(object):
@@ -30,7 +29,6 @@ def __init__(self):
self.skip = False
-
class Folder(Handler):
"""This is the base handler class for any folder handler. It is also used
as the default handler class for any folder resource that has not a more
@@ -41,7 +39,6 @@ class Folder(Handler):
dirty = False
-
def __init__(self, key=None, database=None, **kw):
if database is not None:
self.database = database
@@ -51,7 +48,6 @@ def __init__(self, key=None, database=None, **kw):
if key is not None:
self.key = self.database.normalize_key(key)
-
def get_mtime(self):
"""Returns the last modification time.
"""
@@ -60,7 +56,6 @@ def get_mtime(self):
return None
-
def traverse(self):
yield self
for name in self.get_handler_names():
@@ -71,7 +66,6 @@ def traverse(self):
else:
yield handler
-
def traverse2(self, context=None):
if context is None:
context = Context()
diff --git a/itools/handlers/image.py b/itools/handlers/image.py
index 9dbdf38c1..b00657b30 100644
--- a/itools/handlers/image.py
+++ b/itools/handlers/image.py
@@ -19,7 +19,7 @@
# along with this program. If not, see .
# Import from the Standard Library
-from cStringIO import StringIO
+from io import BytesIO
# Import from the Python Image Library
try:
@@ -38,15 +38,14 @@
rsvg_handle = None
# Import from itools
-from file import File
-from registry import register_handler_class
+from .file import File
+from .registry import register_handler_class
# This number controls the max surface ratio that we can lose when we crop.
MAX_CROP_RATIO = 2.0
-
class Image(File):
class_mimetypes = ['image']
@@ -61,13 +60,12 @@ def _load_state_from_file(self, file):
else:
self.size = (0, 0)
-
def _get_handle(self):
if PIL is False:
return None
# Open image
- f = StringIO(self.data)
+ f = BytesIO(self.data)
try:
im = open_image(f)
except (IOError, OverflowError):
@@ -76,22 +74,18 @@ def _get_handle(self):
# Ok
return im
-
def _get_size(self, handle):
return handle.size
-
def _get_format(self, handle):
return handle.format
-
#########################################################################
# API
#########################################################################
def get_size(self):
return self.size
-
def get_thumbnail(self, xnewsize, ynewsize, format=None, fit=False):
# Get the handle
handle = self._get_handle()
@@ -132,9 +126,9 @@ def get_thumbnail(self, xnewsize, ynewsize, format=None, fit=False):
im, xsize, ysize = self._scale_down(handle, ratio)
# To string
- output = StringIO()
+ output = BytesIO()
# JPEG : Convert to RGB
- if format.lower() == 'jpeg':
+ if format.lower() in ("jpeg", "mpo"):
im = im.convert("RGB")
im.save(output, format, quality=80)
value = output.getvalue()
@@ -143,7 +137,6 @@ def get_thumbnail(self, xnewsize, ynewsize, format=None, fit=False):
# Ok
return value, format.lower()
-
def _scale_down(self, im, ratio):
# Convert to RGBA
im = im.convert("RGBA")
@@ -157,12 +150,10 @@ def _scale_down(self, im, ratio):
return im, xsize, ysize
-
class SVGFile(Image):
class_mimetypes = ['image/svg+xml']
-
def _get_handle(self):
if rsvg_handle is None:
return None
@@ -173,15 +164,12 @@ def _get_handle(self):
svg.close()
return svg
-
def _get_size(self, handle):
return handle.get_property('width'), handle.get_property('height')
-
def _get_format(self, handle):
return 'PNG'
-
def _scale_down(self, handle, ratio):
xsize, ysize = self.size
if ratio >= 1.0:
@@ -206,6 +194,5 @@ def _scale_down(self, handle, ratio):
return im, xsize, ysize
-
register_handler_class(Image)
register_handler_class(SVGFile)
diff --git a/itools/handlers/js.py b/itools/handlers/js.py
index aae207e2b..ea53671eb 100644
--- a/itools/handlers/js.py
+++ b/itools/handlers/js.py
@@ -15,10 +15,8 @@
# along with this program. If not, see .
# Import from itools
-from registry import register_handler_class
-from text import TextFile
-
-
+from .registry import register_handler_class
+from .text import TextFile
class JSFile(TextFile):
@@ -26,7 +24,6 @@ class JSFile(TextFile):
class_mimetypes = ['application/x-javascript', 'application/javascript']
class_extension = 'js'
-
def get_units(self, srx_handler=None):
return []
diff --git a/itools/handlers/registry.py b/itools/handlers/registry.py
index 531523042..db5285e5d 100644
--- a/itools/handlers/registry.py
+++ b/itools/handlers/registry.py
@@ -36,4 +36,4 @@ def get_handler_class_by_mimetype(mimetype, soft=False):
if soft:
return None
- raise ValueError, mimetype
+ raise ValueError(mimetype)
diff --git a/itools/handlers/text.py b/itools/handlers/text.py
old mode 100644
new mode 100755
index ddd687933..5624fb14e
--- a/itools/handlers/text.py
+++ b/itools/handlers/text.py
@@ -16,8 +16,8 @@
# along with this program. If not, see .
# Import from itools
-from file import File
-from registry import register_handler_class
+from .file import File
+from .registry import register_handler_class
def guess_encoding(data):
@@ -25,9 +25,12 @@ def guess_encoding(data):
the wrong encoding, for example many utf8 files will be identified
as latin1.
"""
+ if isinstance(data, bytes):
+ return 'utf8'
+
for encoding in ('ascii', 'utf8', 'iso8859'):
try:
- unicode(data, encoding)
+ data.encode(encoding)
except UnicodeError:
pass
else:
@@ -41,17 +44,16 @@ class TextFile(File):
class_mimetypes = ['text']
class_extension = 'txt'
+ is_text = True
-
- def new(self, data=u''):
+ def new(self, data=''):
self.data = data
self.encoding = 'utf-8'
-
def _load_state_from_file(self, file):
data = file.read()
self.encoding = guess_encoding(data)
- self.data = unicode(data, self.encoding)
+ self.data = data
#########################################################################
@@ -60,18 +62,18 @@ def _load_state_from_file(self, file):
def get_encoding(self):
return self.encoding
-
def to_str(self, encoding='utf-8'):
- return self.data.encode(encoding)
+ # XXX self.data should always be unicode
+ if type(self.data) is bytes:
+ return self.data
+ return self.data.encode(encoding)
def to_text(self):
- return unicode(self.to_str(), 'utf-8')
-
+ return str(self.to_str(), 'utf-8')
def is_empty(self):
- return self.to_text().strip() == u""
-
+ return self.to_text().strip() == ""
register_handler_class(TextFile)
diff --git a/itools/handlers/utils.py b/itools/handlers/utils.py
old mode 100644
new mode 100755
index f9fbe3a0c..fa230be12
--- a/itools/handlers/utils.py
+++ b/itools/handlers/utils.py
@@ -19,60 +19,61 @@
# Import from the Standard Library
import unicodedata
+import sys
+src = (r"""ÄÅÁÀÂÃĀäåáàâãāăÇçÉÈÊËĒéèêëēğÍÌÎÏĪíìîïīıļÑñÖÓÒÔÕØŌöóòôõøōőÜÚÙÛŪüúùûū
+ ŞşšţÝŸȲýÿȳŽž°«»’""")
+dst = (r"""AAAAAAAaaaaaaaaCcEEEEEeeeeegIIIIIiiiiiilNnOOOOOOOooooooooUUUUUuuuuu
+ SsstYYYyyyZz----""")
-src = (ur"""ÄÅÁÀÂÃĀäåáàâãāăÇçÉÈÊËĒéèêëēğÍÌÎÏĪíìîïīıļÑñÖÓÒÔÕØŌöóòôõøōőÜÚÙÛŪüúùûū"""
- ur"""ŞşšţÝŸȲýÿȳŽž°«»’""")
-dst = (ur"""AAAAAAAaaaaaaaaCcEEEEEeeeeegIIIIIiiiiiilNnOOOOOOOooooooooUUUUUuuuuu"""
- ur"""SsstYYYyyyZz----""")
transmap = {}
for i in range(len(src)):
a, b = src[i], dst[i]
transmap[ord(a)] = b
-transmap[ord(u'æ')] = u'ae'
-transmap[ord(u'Æ')] = u'AE'
-transmap[ord(u'œ')] = u'oe'
-transmap[ord(u'Œ')] = u'OE'
-transmap[ord(u'ß')] = u'ss'
+transmap[ord('æ')] = 'ae'
+transmap[ord('Æ')] = 'AE'
+transmap[ord('œ')] = 'oe'
+transmap[ord('Œ')] = 'OE'
+transmap[ord('ß')] = 'ss'
-def checkid(id, soft=True):
+def checkid(_id, soft=True):
"""Turn a bytestring or unicode into an identifier only composed of
alphanumerical characters and a limited list of signs.
It only supports Latin-based alphabets.
"""
- if type(id) is str:
- id = unicode(id, 'utf8')
+ if type(_id) is str:
+ _id = _id
# Normalize unicode
- id = unicodedata.normalize('NFKC', id)
+ _id = unicodedata.normalize('NFKC', _id)
# Strip diacritics
- id = id.strip().translate(transmap)
+ _id = _id.strip().translate(transmap)
# Check for unallowed characters
if soft:
- allowed_characters = set([u'.', u'-', u'_', u'@'])
- id = [ x if (x.isalnum() or x in allowed_characters) else u'-'
- for x in id ]
- id = u''.join(id)
+ allowed_characters = {'.', '-', '_', '@'}
+ _id = [ x if (x.isalnum() or x in allowed_characters) else '-'
+ for x in _id]
+ _id = ''.join(_id)
# Merge hyphens
- id = id.split(u'-')
- id = u'-'.join([x for x in id if x])
- id = id.strip('-')
+ _id = _id.split('-')
+ _id = '-'.join([x for x in _id if x])
+ _id = _id.strip('-')
- # Check wether the id is empty
- if len(id) == 0:
+ # Check wether the _id is empty
+ if len(_id) == 0:
return None
# No mixed case
- id = id.lower()
+ _id = _id.lower()
# Most FS are limited in 255 chars per name
# (keep space for ".metadata" extension)
- id = id[:246]
+ _id = _id[:246]
# Return a safe ASCII bytestring
- return str(id)
+ return str(_id)
diff --git a/itools/html/__init__.py b/itools/html/__init__.py
index 4292663a9..43bd71598 100644
--- a/itools/html/__init__.py
+++ b/itools/html/__init__.py
@@ -19,12 +19,12 @@
# Import from itools
from itools.core import add_type, get_abspath
from itools.xml import register_dtd, DocType
-from filters import sanitize_stream, sanitize_str
-from html import HTMLFile
-from parser import HTMLParser
-from xhtml import XHTMLFile, xhtml_uri, stream_is_empty
-from xhtml import stream_to_str_as_html, stream_to_str_as_xhtml
-import schema
+from . import schema
+from .filters import sanitize_stream, sanitize_str
+from .html import HTMLFile
+from .parser import HTMLParser
+from .xhtml import XHTMLFile, xhtml_uri, stream_is_empty
+from .xhtml import stream_to_str_as_html, stream_to_str_as_xhtml
# Public API
diff --git a/itools/html/filters.py b/itools/html/filters.py
index 456e7eaba..9e147ec4c 100644
--- a/itools/html/filters.py
+++ b/itools/html/filters.py
@@ -78,7 +78,7 @@ def sanitize_stream(stream):
continue
# Filter attributes
attributes = attributes.copy()
- for attr_key in attributes.keys():
+ for attr_key in list(attributes.keys()):
attr_value = attributes[attr_key]
attr_uri, attr_name = attr_key
# Check it is a safe attribute
@@ -114,7 +114,6 @@ def sanitize_stream(stream):
yield event
-
def sanitize_str(str):
stream = XMLParser(str)
return sanitize_stream(stream)
diff --git a/itools/html/html.py b/itools/html/html.py
index 7da62a43b..f3fb1f86f 100644
--- a/itools/html/html.py
+++ b/itools/html/html.py
@@ -18,9 +18,8 @@
# Import from itools
from itools.handlers import register_handler_class
from itools.xmlfile import translate
-from xhtml import XHTMLFile, stream_to_str_as_html
-from parser import HTMLParser
-
+from .xhtml import XHTMLFile, stream_to_str_as_html
+from .parser import HTMLParser
class HTMLFile(XHTMLFile):
@@ -51,22 +50,18 @@ def get_skeleton(cls, title=''):
'