-
Notifications
You must be signed in to change notification settings - Fork 1
/
schema.py
539 lines (457 loc) · 18.4 KB
/
schema.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
# -*- coding: UTF-8 -*-
# Copyright (C) 2005 <jdavid@favela.(none)>
# Copyright (C) 2006 J. David Ibanez <[email protected]>
# Copyright (C) 2006 luis <[email protected]>
# Copyright (C) 2006-2008, 2010 Hervé Cauwelier <[email protected]>
# Copyright (C) 2008 Henry Obein <[email protected]>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# Import from the Standard Library
import re
# Import from itools
from itools.csv import CSVFile
from itools.datatypes import Enumerate, String, Integer, Boolean, Date
from itools.datatypes import Unicode
from itools.gettext import MSG
from itools.web import ERROR, STLView
# Import from ikaaro
from ikaaro.fields import File_Field, Char_Field
from ikaaro.folder import Folder
# Import from goodforms
from datatypes import NumInteger, NumDecimal, NumTime, NumShortTime, Text
from datatypes import NumDate, NumShortDate, NumDigit, UnicodeSQL, EnumBoolean, EmailField
from datatypes import SqlEnumerate, FileImage, Numeric
from utils import SI, FormatError
ERR_BAD_NAME = ERROR(u'In schema, line {line}, variable "{name}" isinvalid.')
ERR_DUPLICATE_NAME = ERROR(u'In schema, line {line}, variable "{name}" is duplicated.')
ERR_BAD_TYPE = ERROR(u'In schema, line {line}, type "{type}" is invalid.')
ERR_BAD_LENGTH = ERROR(u'In schema, line {line}, length "{length}" is invalid.')
ERR_MISSING_OPTIONS = ERROR(u'In schema, line {line}, enum options are missing.')
ERR_BAD_ENUM_REPR = ERROR(u'In schema, line {line}, enum representation "{enum_repr}" is invalid.')
ERR_BAD_DECIMALS = ERROR(u'In schema, line {line}, decimals "{decimals}" are invalid.')
ERR_BAD_MANDATORY = ERROR(u'In schema, line {line}, mandatory "{mandatory}" is invalid.')
ERR_BAD_SIZE = ERROR(u'In schema, line {line}, size ' u'"{size}" is invalid.')
ERR_BAD_DEPENDENCY = ERROR(u'In schema, line {line}, syntax error in dependency: {err}')
ERR_BAD_FORMULA = ERROR(u'In schema, line {line}, syntax error in formula: {err}')
ERR_NO_FORMULA = ERROR(u'In schema, line {line}, type "{type}" does not support formulas.')
ERR_BAD_DEFAULT = ERROR(u'In schema, line {line}, default value "{default}" is invalid.')
class Variable(String):
FIELD_PREFIX = '#'
def decode(cls, data):
data = data.strip().upper()
if not data:
# Turn it into default value at the time of writing
return None
if data[0] == cls.FIELD_PREFIX:
data = data[1:]
return String.decode(data)
def is_valid(cls, value):
return bool(value)
def get_page_number(cls, value):
page_number = ''
for char in value:
if char.isalpha():
page_number += char
else:
break
return page_number
class Type(Enumerate):
options = [
{'name': 'bool', 'value': u"Boolean", 'type': EnumBoolean},
{'name': 'dec', 'value': u"Decimal", 'type': NumDecimal},
{'name': 'digit', 'value': u"00000", 'type': NumDigit},
{'name': 'hh:mm', 'value': u"HH:MM", 'type': NumShortTime},
{'name': 'hhh:mm', 'value': u"HHH:MM", 'type': NumTime},
{'name': 'int', 'value': u"Integer", 'type': NumInteger},
{'name': 'jj/mm/aaaa', 'value': u"DD/MM/YYYY", 'type': NumDate},
{'name': 'mm/aaaa', 'value': u"MM/YYYY", 'type': NumShortDate},
{'name': 'str', 'value': u"String", 'type': UnicodeSQL},
{'name': 'text', 'value': u"Text", 'type': Text},
{'name': 'enum', 'value': u"List of values", 'type': SqlEnumerate},
{'name': 'file', 'value': u"File or Image", 'type': FileImage},
{'name': 'email', 'value': u"Email", 'type': EmailField}]
@classmethod
def decode(cls, data):
data = data.strip().lower()
if not data:
# Turn it into default value at the time of writing
return None
return Enumerate.decode(data)
@classmethod
def get_type(cls, name):
for option in cls.options:
if option['name'] == name:
return option['type']
return None
class ValidInteger(Integer):
def decode(cls, data):
data = data.strip()
if not data:
# Turn it into default value at the time of writing
return None
try:
value = Integer.decode(data)
except ValueError:
value = data
return value
def is_valid(cls, value):
return type(value) is int
class EnumerateOptions(Unicode):
default = None
multiple = True
def decode(cls, data):
value = Unicode.decode(data)
return {
#'name': checkid(value) or '',
'name': value,
'value': value}
def encode(cls, value):
if value is None:
return None
return Unicode.encode(value['value'])
def is_valid(cls, value):
return value is not None or value['name'] is not None
def split(cls, value):
for separator in (u"\r\n", u"\n"):
options = value.split(separator)
if len(options) > 1:
break
else:
# Backwards compatibility
options = [option.strip() for option in value.split(u"/")]
return [{
#'name': checkid(option) or '',
'name': option,
'value': option}
for option in options]
class EnumerateRepresentation(Enumerate):
options = [
{'name': 'select', 'value': MSG(u"Select")},
{'name': 'radio', 'value': MSG(u"Radio")},
{'name': 'checkbox', 'value': MSG(u"Checkbox")}]
def decode(cls, data):
data = data.strip().lower()
if not data:
# Turn it into default value at the time of writing
return None
return Enumerate.decode(data)
class Mandatory(Boolean):
def decode(cls, data):
data = data.strip().upper()
if not data:
# Turn it into default value at the time of writing
return None
elif data in ('O', 'OUI', 'Y', 'YES', '1'):
return True
elif data in ('N', 'NON', 'N', 'NO', '0'):
return False
return data
def is_valid(cls, value):
return type(value) is bool
single_eq = re.compile(ur"""(?<=[^!<>=])[=](?=[^=])""")
class Expression(Unicode):
def decode(cls, data):
# Neither upper() nor lower() to preserve enumerates
value = Unicode.decode(data.strip())
# Allow single "=" as equals
value = single_eq.sub(ur"==", value)
value = (value
# Alternative to name variables
.replace(u'#', u'')
# Non-break spaces
.replace(u'\u00a0', u'')
# Fucking replacement
.replace(u'«', u'"').replace(u'»', u'"'))
return value
def is_simple(cls, value):
if '=' in value or 'in' in value:
return False
return True
def is_valid(cls, value, locals_):
globals_ = {'SI': SI}
try:
eval(value, globals_, locals_)
except ZeroDivisionError:
pass
except Exception:
# Let error raise with message
raise
return True
class SchemaHandler(CSVFile):
# Don't store default values here because any value needs to be written
# down in case the default value changes later.
skip_header = True
has_header = True
class_csv_guess = True
schema = {
'title': Unicode(mandatory=True, title=MSG(u"Title")),
'name': Variable(mandatory=True, title=MSG(u"Variable")),
'type': Type(mandatory=True, title=MSG(u"Type")),
'help': Unicode(title=MSG(u"Online Help")),
'length': ValidInteger(title=MSG(u"Length")),
'enum_options': EnumerateOptions(mandatory=True,
title=MSG(u"Enumerate Options")),
'enum_repr': EnumerateRepresentation(
title=MSG(u"Enumerate Representation")),
'decimals': ValidInteger(title=MSG(u"Decimals")),
'mandatory': Mandatory(title=MSG(u"Mandatory")),
'size': ValidInteger(title=MSG(u"Input Size")),
'dependency': Expression(title=MSG(u"Dependent Field")),
'formula': Expression(title=MSG(u"Formula")),
'default': String(default='', title=MSG(u"Default Value"))}
columns = [
'title',
'name',
'type',
'help',
'length',
'enum_options',
'enum_repr',
'decimals',
'mandatory',
'size',
'dependency',
'formula',
'default']
class Schema_DebugView(STLView):
access = 'is_admin'
template = '/ui/goodforms/schema_debug.xml'
def get_namespace(self, resource, context):
errors = resource.get_errors()
if errors:
schema = {}
else:
schema, pages = resource.get_schema_pages()
return {'schema': schema.keys(),
'errors': errors}
class Schema(Folder):
class_id = 'schema'
class_title = MSG(u"Schema")
class_views = ['debug']
class_icon_css = 'fa-bars'
# Fields
data = File_Field(class_handler=SchemaHandler)
extension = Char_Field
filename = Char_Field
def _load_from_csv(self):
errors = self.get_errors()
if errors:
error = errors[0]
raise FormatError(error)
def get_errors(self):
errors = []
handler = self.get_value('data')
# Consistency check
# First round on variables
# Starting from 1 + header
lineno = 2
locals_ = {}
for line in handler.get_rows():
record = {}
for index, key in enumerate(handler.columns):
record[key] = line[index]
# Name
name = record['name']
if name is None:
continue
if not Variable.is_valid(name):
err = ERR_BAD_NAME.gettext(line=lineno, name=name)
errors.append(err)
continue
if name in locals_:
err = ERR_DUPLICATE_NAME.gettext(line=lineno, name=name)
errors.append(err)
continue
# Type
type_name = record['type']
if type_name is None:
# Write down default at this time
record['type'] = type_name = 'str'
datatype = Type.get_type(type_name)
if datatype is None:
err = ERR_BAD_TYPE.gettext(line=lineno, type=type_name)
errors.append(err)
continue
# Length
length = record['length']
if length is None:
# Write down default at this time
record['length'] = length = 20
if not ValidInteger.is_valid(length):
err = ERR_BAD_LENGTH.gettext(line=lineno, length=length)
errors.append(err)
continue
if issubclass(datatype, SqlEnumerate):
# Enumerate Options
enum_option = record['enum_options']
if enum_option is None:
err = ERR_MISSING_OPTIONS(line=lineno)
errors.append(err)
continue
# Split on "/"
enum_options = EnumerateOptions.split(enum_option['value'])
record['enum_options'] = enum_options
# Enumerate Representation
enum_repr = record['enum_repr']
if enum_repr is None:
# Write down default at the time of writing
record['enum_repr'] = enum_repr = 'radio'
if not EnumerateRepresentation.is_valid(enum_repr):
err = ERR_BAD_ENUM_REPR.gettext(line=lineno, enum_repr=enum_repr)
errors.append(err)
continue
elif issubclass(datatype, NumDecimal):
# Decimals
decimals = record['decimals']
if decimals is None:
# Write down default at the time of writing
record['decimals'] = decimals = 2
if not ValidInteger.is_valid(decimals):
err = ERR_BAD_DECIMALS.gettext(line=lineno, decimals=decimals)
errors.append(err)
continue
# Mandatory
mandatory = record['mandatory']
if mandatory is None:
# Write down default at the time of writing
record['mandatory'] = mandatory = True
if not Mandatory.is_valid(mandatory):
err = ERR_BAD_MANDATORY.gettext(line=lineno, mandatory=mandatory)
errors.append(err)
continue
# Size
size = record['size']
if size is None:
# Write down default at the time of writing
if type_name == 'text':
record['size'] = size = 5
else:
record['size'] = size = length
if not ValidInteger.is_valid(size):
err = ERR_BAD_SIZE.gettext(line=lineno, size=size)
errors.append(err)
continue
# Default value
default = record['default'] = record['default'].strip()
if default:
if issubclass(datatype, EnumBoolean):
value = Mandatory.decode(default)
default = EnumBoolean.encode(value)
elif issubclass(datatype, SqlEnumerate):
datatype = datatype(options=enum_options)
#default = checkid(default) or ''
default = default
elif issubclass(datatype, NumTime):
# "0-0-0 09:00:00" -> "09:00:00"
default = default.split(' ')[-1]
# "09:00:00" -> "09:00"
if default.count(":") > 1:
default = default.rsplit(":", 1)[0]
elif issubclass(datatype, NumDate):
# "2010-11-18 00:00:00" -> "18/11/2010"
default = default.split(' ')[0]
value = Date.decode(default)
default = NumDate.encode(value)
elif issubclass(datatype, NumDigit):
datatype = datatype(length=length)
if not datatype.is_valid(default):
err = ERR_BAD_DEFAULT.gettext(line=lineno, default=unicode(default, 'utf_8'))
errors.append(err)
continue
record['default'] = default
if record['enum_repr'] == 'checkbox':
locals_[name] = []
else:
locals_[name] = 0
lineno += 1
# Second round on references
# Starting from 1 + header
lineno = 2
for row in handler.get_rows():
dependency = row.get_value('dependency')
if dependency:
try:
Expression.is_valid(dependency, locals_)
except Exception, err:
err = ERR_BAD_DEPENDENCY.gettext(line=lineno, err=err)
errors.append(err)
continue
formula = row.get_value('formula')
if formula:
try:
datatype.sum
except AttributeError:
err = ERR_NO_FORMULA.gettext(line=lineno, type=type_name)
errors.append(err)
continue
try:
Expression.is_valid(formula, locals_)
except Exception, err:
err = ERR_BAD_FORMULA.gettext(line=lineno, err=err)
errors.append(err)
continue
lineno += 1
# Ok
return errors
def get_schema_pages(self):
schema = {}
pages = {}
handler = self.get_value('data')
for row in handler.get_rows():
# The name
name = row.get_value('name')
# The datatype
type_name = row.get_value('type')
datatype = Type.get_type(type_name)
multiple = False
# TypeError: issubclass() arg 1 must be a class
if isinstance(datatype, Numeric):
pass
elif issubclass(datatype, SqlEnumerate):
enum_options = row.get_value('enum_options')
representation = row.get_value('enum_repr')
multiple = (representation == 'checkbox')
datatype = datatype(options=enum_options,
representation=representation)
elif issubclass(datatype, EnumBoolean):
datatype = datatype(representation='radio')
multiple = False
# The page number (now automatic)
page_number = Variable.get_page_number(name)
pages.setdefault(page_number, set()).add(name)
page_numbers = (page_number,)
# Add to the datatype
default = row.get_value('default')
if multiple:
default = [default]
length = row.get_value('length')
size = row.get_value('size') or length
schema[name] = datatype(multiple=multiple,
type=type_name,
default=datatype.decode(default),
# Read only for Scrib
readonly=False,
pages=page_numbers,
title=row.get_value('title'),
help=row.get_value('help'),
length=length,
decimals=row.get_value('decimals'),
mandatory=row.get_value('mandatory'),
size=size,
dependency=row.get_value('dependency'),
formula=row.get_value('formula'))
return schema, pages
# Views
debug = Schema_DebugView()