From fd0b1654d85458a0209bd8808036c4ac9056f6a2 Mon Sep 17 00:00:00 2001 From: Thomas Buchberger Date: Wed, 16 Oct 2024 10:37:21 +0200 Subject: [PATCH 1/5] Export NG [WiP] --- opengever/exportng/__init__.py | 0 opengever/exportng/db.py | 43 +++++ opengever/exportng/mapping.py | 21 +++ opengever/exportng/sync.py | 299 +++++++++++++++++++++++++++++++++ opengever/exportng/zopectl.py | 86 ++++++++++ setup.py | 1 + 6 files changed, 450 insertions(+) create mode 100644 opengever/exportng/__init__.py create mode 100644 opengever/exportng/db.py create mode 100644 opengever/exportng/mapping.py create mode 100644 opengever/exportng/sync.py create mode 100644 opengever/exportng/zopectl.py diff --git a/opengever/exportng/__init__.py b/opengever/exportng/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/opengever/exportng/db.py b/opengever/exportng/db.py new file mode 100644 index 00000000000..ca9ba181d4e --- /dev/null +++ b/opengever/exportng/db.py @@ -0,0 +1,43 @@ +from sqlalchemy import create_engine +from sqlalchemy import MetaData +from sqlalchemy import Table +from sqlalchemy import Column +from sqlalchemy import String +from sqlalchemy import Text +from sqlalchemy import Integer +from sqlalchemy import Date +from sqlalchemy import DateTime +from sqlalchemy import Boolean +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import func + + +COLUMN_TYPES = { + 'varchar': String, + 'text': Text, + 'jsonb': JSONB, + 'integer': Integer, + 'date': Date, + 'datetime': DateTime, + 'boolean': Boolean, +} + +engine = create_engine('postgresql:///exportng') +metadata = MetaData(bind=engine) + + +def create_table(table, mapping): + if table in metadata.tables: + return + cols = [] + for attr in mapping: + cols.append(Column( + attr.col_name, + COLUMN_TYPES[attr.col_type], + nullable=True, + )) + cols.append(Column('_created_at', DateTime, index=True, server_default=func.now())) + cols.append(Column('_modified_at', DateTime, index=True, onupdate=func.now())) + cols.append(Column('_synced_at', DateTime, index=True, onupdate=func.now())) + cols.append(Column('_deleted', Boolean, index=True, default=False)) + Table(table, metadata, *cols) diff --git a/opengever/exportng/mapping.py b/opengever/exportng/mapping.py new file mode 100644 index 00000000000..bc39af11c5f --- /dev/null +++ b/opengever/exportng/mapping.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from openpyxl import load_workbook +import itertools +import os.path + + +def read_mapping(mapping_file): + filename = os.path.join( + os.path.dirname(__file__), mapping_file) + wb = load_workbook(filename=filename) + sheet = wb.active + rows = [] + colnames = [cell.value for cell in sheet[1]] + for row in sheet.iter_rows(min_row=2): + data = {k: row[idx].value for idx, k in enumerate(colnames)} + rows.append(data) + + mappings = {} + for k, v in itertools.groupby(rows, lambda x: x['table_name']): + mappings[k] = list(v) + return mappings diff --git a/opengever/exportng/sync.py b/opengever/exportng/sync.py new file mode 100644 index 00000000000..f755ff314df --- /dev/null +++ b/opengever/exportng/sync.py @@ -0,0 +1,299 @@ +import opengever.ogds.base # noqa fix cyclic import +from Acquisition import aq_parent +from plone import api +from opengever.exportng.db import metadata +from opengever.exportng.db import engine +from opengever.exportng.db import create_table +from opengever.ogds.models.service import ogds_service +from sqlalchemy.sql.expression import false +from sqlalchemy import bindparam +from collections import namedtuple +from opengever.ogds.models.user import User +from opengever.ogds.models.group import Group +from plone.dexterity.utils import iterSchemata +from zope.schema import getFields +import logging +import os.path + + +logger = logging.getLogger('opengever.exportng') + + +Attribute = namedtuple( + 'Attribute', + ['name', 'col_name', 'col_type', 'getter'], +) + +CACHE = {} + + +def rename_dict_key(dict_, old_key, new_key): + dict_[new_key] = dict_.pop(old_key) + return dict_ + + +def dexterity_field_value(obj, attrname): + fields = CACHE.get('dexterity_fields', {}).get(obj.portal_type, None) + if fields is None: + fields = {} + for schema in iterSchemata(obj): + fields.update(getFields(schema)) + CACHE.setdefault('dexterity_fields', {})[obj.portal_type] = fields + field = fields.get(attrname) + return getattr(field.interface(obj), attrname) + + +def as_datetime(obj, attrname): + value = getattr(obj, attrname) + if callable(value): + value = value() + return value.asdatetime() + + +def get_filedata(obj, attrname): + value = getattr(obj, attrname) + if value is not None: + return { + "filepath": value._blob.committed(), + "filename": value.filename, + "mime_type": value.contentType, + } + return None + + +def get_file_extension(obj, attrname): + value = getattr(obj, attrname) + if value is not None: + return os.path.splitext(value.filename)[-1][1:] + + +def userid_to_email(obj, attrname): + userid = dexterity_field_value(obj, attrname) + userid_email_mapping = CACHE.get('userid_email_mapping', None) + if userid_email_mapping is None: + users = ogds_service().all_users() + userid_email_mapping = {user.userid: user.email for user in users} + CACHE['userid_email_mapping'] = userid_email_mapping + return userid_email_mapping.get(userid, userid) + + +def parent_uid(obj, attrname): + return aq_parent(obj).UID() + + +class CatalogSyncer(object): + + catalog_key = 'UID' + sql_key = 'objexternalkey' + + def get_catalog_items(self): + ct = api.portal.get_tool('portal_catalog') + return ct.unrestrictedSearchResults(**self.query) + + def get_sql_items(self): + table = metadata.tables[self.table] + stmt = table.select().where(table.c._deleted == false()) + with engine.connect() as conn: + return conn.execute(stmt).fetchall() + + def sync(self): + catalog_items = self.get_catalog_items() + sql_items = self.get_sql_items() + catalog_keys = set([ + getattr(item, self.catalog_key) for item in catalog_items]) + sql_keys = set([getattr(item, self.sql_key) for item in sql_items]) + catalog_items_by_key = { + getattr(item, self.catalog_key): item for item in catalog_items} + sql_items_by_key = { + getattr(item, self.sql_key): item for item in sql_items} + + added = catalog_keys - sql_keys + deleted = sql_keys - catalog_keys + existing = catalog_keys & sql_keys + + modified = set() + for key in existing: + if ( + catalog_items_by_key[key].modified.asdatetime().replace(tzinfo=None) + != sql_items_by_key[key].modified + ): + modified.add(key) + + inserts = [] + for key in added: + item = catalog_items_by_key[key] + obj = item.getObject() + inserts.append(self.get_values(obj)) + table = metadata.tables[self.table] + with engine.connect() as conn: + conn.execute(table.insert(), inserts) + logger.info('Added %s: %s', table, len(added)) + + updates = [] + for key in modified: + item = catalog_items_by_key[key] + obj = item.getObject() + updates.append(rename_dict_key(self.get_values(obj), self.sql_key, 'b_key')) + if updates: + with engine.connect() as conn: + conn.execute(table.update().where(getattr(table.c, self.sql_key) == bindparam('b_key')), updates) + logger.info('Updated %s: %s', table, len(modified)) + + deletes = [] + for key in deleted: + deletes.append({'b_key': key, '_deleted': True}) + if deletes: + with engine.connect() as conn: + conn.execute(table.update().where( + getattr(table.c, self.sql_key) == bindparam('b_key')), deletes) + logger.info('Deleted %s: %s', table, len(deleted)) + + def get_values(self, obj): + data = {} + # serializer = queryMultiAdapter( + # (self.context, self.request), ISerializeToJson) + # obj_data = serializer(include_items=False) + for attr in self.mapping: + if attr.getter is not None: + value = attr.getter(obj, attr.name) + else: + try: + value = getattr(obj, attr.name) + except AttributeError: + value = None + if callable(value): + value = value() + data[attr.col_name] = value + return data + + def get_fields(self, obj): + fields = {} + for schema in iterSchemata(obj): + fields.update(getFields(schema)) + return fields + + +class DossierSyncer(CatalogSyncer): + + table = 'ogdossiers' + query = { + 'portal_type': 'opengever.dossier.businesscasedossier', + 'is_subdossier': False, + } + mapping = [ + Attribute('UID', 'objexternalkey', 'varchar', None), + Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), + Attribute('title', 'botitle', 'varchar', None), + Attribute('description', 'bodescription', 'varchar', None), + # Attribute('modified', 'modified', 'datetime', as_datetime), + # Attribute('start', 'objvalidfrom', 'date', None), + # Attribute('end', 'objvaliduntil', 'date', None), + Attribute('responsible', 'gboresponsible', 'varchar', userid_to_email), + # Attribute('external_reference', 'boforeignnumber', 'varchar', None), + ] + + +class SubdossierSyncer(DossierSyncer): + + table = 'ogsubdossiers' + query = { + 'portal_type': 'opengever.dossier.businesscasedossier', + 'is_subdossier': True, + } + + +class DocumentSyncer(CatalogSyncer): + + table = 'ogdocuments' + query = { + 'portal_type': 'opengever.document.document', + } + mapping = [ + Attribute('UID', 'objexternalkey', 'varchar', None), + Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), + Attribute('title', 'objname', 'varchar', None), + Attribute('extension', 'extension', 'varchar', get_file_extension), + # Attribute('description', 'bodescription', 'varchar', None), + # Attribute('document_date', 'dadate', 'date', None), + # Attribute('receipt_date', 'gcreceiptdate', 'date', None), + # Attribute('document_author', 'gcauthor', 'varchar', None), + # Attribute('external_reference', 'gcexternalreference', 'varchar', None), + Attribute('file', '_file', 'jsonb', get_filedata), + ] + + +class OGDSSyncer(object): + + def get_sql_items(self): + table = metadata.tables[self.table] + stmt = table.select().where(table.c._deleted == false()) + with engine.connect() as conn: + return conn.execute(stmt).fetchall() + + def get_ogds_items(self): + return self.model.query.all() + + def get_values(self, item): + data = {} + for attr in self.mapping: + data[attr.col_name] = getattr(item, attr.name) + return data + + def truncate(self): + with engine.begin() as conn: + conn.execution_options(autocommit=True).execute( + "TRUNCATE TABLE {}".format(self.table)) + + def sync(self): + self.truncate() + inserts = [] + for item in self.get_ogds_items(): + inserts.append(self.get_values(item)) + + table = metadata.tables[self.table] + with engine.connect() as conn: + conn.execute(table.insert(), inserts) + logger.info('Added %s: %s', table, len(inserts)) + + +class UserSyncer(OGDSSyncer): + + table = 'users' + model = User + + mapping = [ + Attribute('email', 'email', 'varchar', None), + Attribute('firstname', 'firstname', 'varchar', None), + Attribute('lastname', 'lastname', 'varchar', None), + Attribute('active', 'active', 'boolean', None), + ] + + +class GroupSyncer(OGDSSyncer): + + table = 'groups' + model = Group + + mapping = [ + Attribute('groupname', 'groupname', 'varchar', None), + Attribute('title', 'title', 'varchar', None), + Attribute('active', 'active', 'boolean', None), + ] + + +class Syncer(object): + + def create_tables(self): + create_table(UserSyncer.table, UserSyncer.mapping) + create_table(GroupSyncer.table, GroupSyncer.mapping) + create_table(DossierSyncer.table, DossierSyncer.mapping) + create_table(SubdossierSyncer.table, SubdossierSyncer.mapping) + create_table(DocumentSyncer.table, DocumentSyncer.mapping) + metadata.create_all(checkfirst=True) + + def sync(self): + UserSyncer().sync() + GroupSyncer().sync() + DossierSyncer().sync() + SubdossierSyncer().sync() + DocumentSyncer().sync() diff --git a/opengever/exportng/zopectl.py b/opengever/exportng/zopectl.py new file mode 100644 index 00000000000..1cce8255a4f --- /dev/null +++ b/opengever/exportng/zopectl.py @@ -0,0 +1,86 @@ +from opengever.exportng.db import metadata +from opengever.exportng.sync import Syncer +from zope.component.hooks import setSite +from zope.globalrequest import setRequest +import argparse +import logging +import sys + + +logger = logging.getLogger('opengever.exportng') + + +def exportng(app, args): + parser = argparse.ArgumentParser(description="Export data") + parser.add_argument( + '--mapping', + help='Name of the mapping file (Excel)', + default='OGG-Mapping.xlsx', + ) + parser.add_argument( + '--drop-tables', '-D', + help='Drop all tables', + action='store_true', + ) + parser.add_argument( + '--site-path', '-s', + help='Path to the Plone site.', + default=None, + ) + + # If run with plone.recipe.zope2instance we need to strip the first 2 args + if sys.argv[0] != 'exportng': + args = args[2:] + options = parser.parse_args(args) + + setup_logging() + app = setup_request(app) + site = get_site(app, options.site_path) + setSite(site) + + # mapping = read_mapping(options.mapping) + # create_or_reflect_tables(mapping, drop=options.drop_tables) + metadata.reflect() + if options.drop_tables: + metadata.drop_all() + metadata.clear() + + syncer = Syncer() + syncer.create_tables() + syncer.sync() + + +def setup_logging(): + # Set Zope's default StreamHandler's level to INFO (default is WARNING) + # to make sure output gets logged on console + stream_handler = logging.root.handlers[0] + stream_handler.setLevel(logging.INFO) + logging.root.setLevel(logging.INFO) + + +def setup_request(app): + # Delay import of the Testing module + # Importing it before the database is opened, will result in opening a + # DemoStorage database instead of the one from the config file. + from Testing.makerequest import makerequest + app = makerequest(app) + setRequest(app.REQUEST) + return app + + +def get_site(app, site_path): + if site_path is not None: + return app.unrestrictedTraverse(site_path) + else: + sites = [] + for item in app.values(): + if item.meta_type == 'Plone Site': + sites.append(item) + if len(sites) == 1: + return sites[0] + elif len(sites) > 1: + logger.info('Multiple Plone site found.') + sys.exit(1) + else: + logger.info('No Plone site found.') + sys.exit(1) diff --git a/setup.py b/setup.py index 53dead3f692..55c09c9d93b 100644 --- a/setup.py +++ b/setup.py @@ -208,6 +208,7 @@ sync_ogds = opengever.ogds.base:sync_ogds_zopectl_handler setup = opengever.setup.zopectl:setup upgrade = opengever.setup.zopectl:upgrade + exportng = opengever.exportng.zopectl:exportng [console_scripts] create-bundle = opengever.bundle.factory:main From 7529a627d52e26dc21afc360eb4d35d4935cc69f Mon Sep 17 00:00:00 2001 From: Thomas Buchberger Date: Thu, 21 Nov 2024 15:55:53 +0100 Subject: [PATCH 2/5] Add fileplanentry --- opengever/exportng/sync.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/opengever/exportng/sync.py b/opengever/exportng/sync.py index f755ff314df..8b8aa4140fc 100644 --- a/opengever/exportng/sync.py +++ b/opengever/exportng/sync.py @@ -1,6 +1,7 @@ import opengever.ogds.base # noqa fix cyclic import from Acquisition import aq_parent from plone import api +from opengever.base.interfaces import IReferenceNumber from opengever.exportng.db import metadata from opengever.exportng.db import engine from opengever.exportng.db import create_table @@ -62,7 +63,10 @@ def get_filedata(obj, attrname): def get_file_extension(obj, attrname): - value = getattr(obj, attrname) + if obj.portal_type == 'opengever.document.document': + value = getattr(obj, 'file') + else: + value = getattr(obj, 'message') if value is not None: return os.path.splitext(value.filename)[-1][1:] @@ -77,6 +81,10 @@ def userid_to_email(obj, attrname): return userid_email_mapping.get(userid, userid) +def get_reference_number(obj, attrname): + return '.'.join(IReferenceNumber(obj).get_numbers()['repository']) + + def parent_uid(obj, attrname): return aq_parent(obj).UID() @@ -173,6 +181,26 @@ def get_fields(self, obj): return fields +class FileplanEntrySyncer(CatalogSyncer): + + table = 'fileplanentries' + query = { + 'portal_type': 'opengever.repository.repositoryfolder', + } + mapping = [ + Attribute('UID', 'objexternalkey', 'varchar', None), + Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), + Attribute('title', 'botitle', 'varchar', None), + Attribute('description', 'bodescription', 'varchar', None), + Attribute('location', 'felocation', 'varchar', None), + Attribute('reference', 'fcsbusinessnumber', 'varchar', get_reference_number), + # Attribute('modified', 'modified', 'datetime', as_datetime), + Attribute('valid_from', 'objvalidfrom', 'date', None), + Attribute('valid_until', 'objvaliduntil', 'date', None), + # Attribute('external_reference', 'boforeignnumber', 'varchar', None), + ] + + class DossierSyncer(CatalogSyncer): table = 'ogdossiers' @@ -286,6 +314,7 @@ class Syncer(object): def create_tables(self): create_table(UserSyncer.table, UserSyncer.mapping) create_table(GroupSyncer.table, GroupSyncer.mapping) + create_table(FileplanEntrySyncer.table, FileplanEntrySyncer.mapping) create_table(DossierSyncer.table, DossierSyncer.mapping) create_table(SubdossierSyncer.table, SubdossierSyncer.mapping) create_table(DocumentSyncer.table, DocumentSyncer.mapping) @@ -294,6 +323,7 @@ def create_tables(self): def sync(self): UserSyncer().sync() GroupSyncer().sync() + FileplanEntrySyncer().sync() DossierSyncer().sync() SubdossierSyncer().sync() DocumentSyncer().sync() From cda1695a0d2afc24591d25611cc7264af06310bc Mon Sep 17 00:00:00 2001 From: Philippe Gross Date: Thu, 21 Nov 2024 15:33:56 +0100 Subject: [PATCH 3/5] Extend dossiersyncer attributes. --- opengever/exportng/sync.py | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/opengever/exportng/sync.py b/opengever/exportng/sync.py index f755ff314df..cae4c5451e2 100644 --- a/opengever/exportng/sync.py +++ b/opengever/exportng/sync.py @@ -81,6 +81,48 @@ def parent_uid(obj, attrname): return aq_parent(obj).UID() +def str_upper(obj, attrname): + return getattr(obj, attrname, '').upper() + + +def get_public_trial(obj, attrname): + value_mapping = { + 'unchecked': 'NOTASSESSED', + 'public': 'PUBLIC', + 'limited-public': 'LIMITEDPUBLIC', + 'private': 'PRIVATE', + } + return value_mapping.get(obj.public_trial) + + +def get_archival_value(obj, attrname): + value_mapping = { + 'unchecked': 'NOTASSESSED', + 'prompt': 'PROMPT', + 'archival worthy': 'ARCHIVALWORTHY', + 'not archival worthy': 'NOTARCHIVALWORTHY', + 'archival worthy with sampling': 'SAMPLING' + } + return value_mapping.get(obj.archival_value) + + +def get_privacy_layer(obj, attrname): + value_mapping = { + 'privacy_layer_yes': True, + 'privacy_layer_no': False + } + return value_mapping.get(obj.privacy_layer) + + +def get_dossier_state(obj, attrname): + state_mapping = { + 'dossier-state-active': 'EDIT', + 'dossier-state-inactive': 'CANCELLED', + 'dossier-state-resolved': 'CLOSED' + } + return state_mapping.get(api.content.get_state(obj)) + + class CatalogSyncer(object): catalog_key = 'UID' @@ -183,13 +225,31 @@ class DossierSyncer(CatalogSyncer): mapping = [ Attribute('UID', 'objexternalkey', 'varchar', None), Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), + # Attribute('modified', 'modified', 'datetime', as_datetime), + # Attribute('changed', 'changed', 'datetime', None) + # Attribute('touched', 'touched', 'datetime', None) Attribute('title', 'botitle', 'varchar', None), Attribute('description', 'bodescription', 'varchar', None), - # Attribute('modified', 'modified', 'datetime', as_datetime), + Attribute('Creator', 'objcreatedby', 'varchar', None), + Attribute('review_sate', 'bostate', 'varchar', get_dossier_state), + # Attribute('keywords', 'keywords', 'varchar', None), # Attribute('start', 'objvalidfrom', 'date', None), # Attribute('end', 'objvaliduntil', 'date', None), Attribute('responsible', 'gboresponsible', 'varchar', userid_to_email), # Attribute('external_reference', 'boforeignnumber', 'varchar', None), + # Attribute('relatedDossier', 'XXX', 'varchar', None), + # Attribute('former_reference_number', 'bonumberhistory', 'varchar', None), + # Attribute('reference_number', 'bonumberhistory', 'varchar', None), + # Attribute('dossier_type', 'dossier_type', 'varchar', None), + Attribute('classification', 'classification', 'varchar', str_upper), + Attribute('privacy_layer', 'XX', 'varchar', get_privacy_layer), + Attribute('public_trial', 'disclosurestatus', 'varchar', get_public_trial), + Attribute('public_trial_statement', 'disclosurestatusstatement', 'varchar', None), + Attribute('retention_period', 'retentionperiod', 'integer', None), + Attribute('retention_period_annotation', 'retentionperiodcomment', 'varchar', None), + Attribute('archival_value', 'archivalvalue', 'varchar', get_archival_value), + Attribute('archival_value_annotation', 'archivalvaluecomment', 'varchar', None), + Attribute('custody_period', 'regularsafeguardperiod', 'varchar', None), ] From a7711155dc9c95a501594b17d99c255de5ffdd24 Mon Sep 17 00:00:00 2001 From: Philippe Gross Date: Mon, 25 Nov 2024 12:14:38 +0100 Subject: [PATCH 4/5] Extend document syncer attributes. --- opengever/exportng/sync.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/opengever/exportng/sync.py b/opengever/exportng/sync.py index cae4c5451e2..5b484fe0fd5 100644 --- a/opengever/exportng/sync.py +++ b/opengever/exportng/sync.py @@ -225,7 +225,7 @@ class DossierSyncer(CatalogSyncer): mapping = [ Attribute('UID', 'objexternalkey', 'varchar', None), Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), - # Attribute('modified', 'modified', 'datetime', as_datetime), + Attribute('modified', 'objmodifieddate', 'datetime', as_datetime), # Attribute('changed', 'changed', 'datetime', None) # Attribute('touched', 'touched', 'datetime', None) Attribute('title', 'botitle', 'varchar', None), @@ -233,8 +233,8 @@ class DossierSyncer(CatalogSyncer): Attribute('Creator', 'objcreatedby', 'varchar', None), Attribute('review_sate', 'bostate', 'varchar', get_dossier_state), # Attribute('keywords', 'keywords', 'varchar', None), - # Attribute('start', 'objvalidfrom', 'date', None), - # Attribute('end', 'objvaliduntil', 'date', None), + Attribute('start', 'objvalidfrom', 'date', None), + Attribute('end', 'objvalidto', 'date', None), Attribute('responsible', 'gboresponsible', 'varchar', userid_to_email), # Attribute('external_reference', 'boforeignnumber', 'varchar', None), # Attribute('relatedDossier', 'XXX', 'varchar', None), @@ -242,7 +242,7 @@ class DossierSyncer(CatalogSyncer): # Attribute('reference_number', 'bonumberhistory', 'varchar', None), # Attribute('dossier_type', 'dossier_type', 'varchar', None), Attribute('classification', 'classification', 'varchar', str_upper), - Attribute('privacy_layer', 'XX', 'varchar', get_privacy_layer), + Attribute('privacy_layer', 'privacyprotection', 'varchar', get_privacy_layer), Attribute('public_trial', 'disclosurestatus', 'varchar', get_public_trial), Attribute('public_trial_statement', 'disclosurestatusstatement', 'varchar', None), Attribute('retention_period', 'retentionperiod', 'integer', None), @@ -272,13 +272,23 @@ class DocumentSyncer(CatalogSyncer): Attribute('UID', 'objexternalkey', 'varchar', None), Attribute('parent', 'objprimaryrelated', 'varchar', parent_uid), Attribute('title', 'objname', 'varchar', None), - Attribute('extension', 'extension', 'varchar', get_file_extension), - # Attribute('description', 'bodescription', 'varchar', None), - # Attribute('document_date', 'dadate', 'date', None), - # Attribute('receipt_date', 'gcreceiptdate', 'date', None), - # Attribute('document_author', 'gcauthor', 'varchar', None), - # Attribute('external_reference', 'gcexternalreference', 'varchar', None), + Attribute('Creator', 'objcreatedby', 'varchar', None), Attribute('file', '_file', 'jsonb', get_filedata), + Attribute('extension', 'extension', 'varchar', get_file_extension), + # Attribute('changed', 'changed', 'datetime', None) + Attribute('privacy_layer', 'privacyprotection', 'varchar', get_privacy_layer), + Attribute('public_trial', 'disclosurestatus', 'varchar', get_public_trial), + Attribute('public_trial_statement', 'disclosurestatusstatement', 'varchar', None), + # Attribute('relatedItems', 'XXX', 'varchar', None), + Attribute('description', 'dadescription', 'varchar', None), + # Attribute('keywords', 'XXX', 'varchar', None), + Attribute('foreign_reference', 'gcexternalreference', 'varchar', None), + Attribute('document_date', 'dadate', 'date', None), + Attribute('receipt_date', 'gcreceiptdate', 'date', None), + Attribute('delivery_date', 'gcdeliverydate', 'date', None), + # Attribute('document_type', 'XXX', 'date', None), + Attribute('document_author', 'gcauthor', 'varchar', None), + # Attribute('preserved_as_paper', 'XXX', 'varchar', None), ] From ef605d2cc5702794f17d27d11c101a30ca3a5f1e Mon Sep 17 00:00:00 2001 From: Philippe Gross Date: Wed, 27 Nov 2024 10:33:00 +0100 Subject: [PATCH 5/5] QA --- opengever/exportng/db.py | 12 ++++++------ opengever/exportng/sync.py | 17 ++++++++--------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/opengever/exportng/db.py b/opengever/exportng/db.py index ca9ba181d4e..765ca5b6f8f 100644 --- a/opengever/exportng/db.py +++ b/opengever/exportng/db.py @@ -1,13 +1,13 @@ +from sqlalchemy import Boolean +from sqlalchemy import Column from sqlalchemy import create_engine +from sqlalchemy import Date +from sqlalchemy import DateTime +from sqlalchemy import Integer from sqlalchemy import MetaData -from sqlalchemy import Table -from sqlalchemy import Column from sqlalchemy import String +from sqlalchemy import Table from sqlalchemy import Text -from sqlalchemy import Integer -from sqlalchemy import Date -from sqlalchemy import DateTime -from sqlalchemy import Boolean from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.sql import func diff --git a/opengever/exportng/sync.py b/opengever/exportng/sync.py index 5b484fe0fd5..def87a62507 100644 --- a/opengever/exportng/sync.py +++ b/opengever/exportng/sync.py @@ -1,21 +1,20 @@ -import opengever.ogds.base # noqa fix cyclic import from Acquisition import aq_parent -from plone import api -from opengever.exportng.db import metadata -from opengever.exportng.db import engine +from collections import namedtuple from opengever.exportng.db import create_table +from opengever.exportng.db import engine +from opengever.exportng.db import metadata +from opengever.ogds.models.group import Group from opengever.ogds.models.service import ogds_service -from sqlalchemy.sql.expression import false -from sqlalchemy import bindparam -from collections import namedtuple from opengever.ogds.models.user import User -from opengever.ogds.models.group import Group +from plone import api from plone.dexterity.utils import iterSchemata +from sqlalchemy import bindparam +from sqlalchemy.sql.expression import false from zope.schema import getFields import logging +import opengever.ogds.base # noqa fix cyclic import import os.path - logger = logging.getLogger('opengever.exportng')