diff --git a/account_statement_import_sheet_file/README.rst b/account_statement_import_sheet_file/README.rst index b661fdcfc..864095c0e 100644 --- a/account_statement_import_sheet_file/README.rst +++ b/account_statement_import_sheet_file/README.rst @@ -103,6 +103,8 @@ Contributors * Alexey Pelykh +* Sebastiano Picchi + Maintainers ~~~~~~~~~~~ diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py index 2db125538..34f996b45 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py @@ -66,6 +66,18 @@ class AccountStatementImportSheetMapping(models.Model): help="When this occurs please indicate the column number in the Columns section " "instead of the column name, considering that the first column is 0", ) + skip_empty_lines = fields.Boolean( + default=False, + help="Allows to skip empty lines", + ) + offset_column = fields.Integer( + default=0, + help="Horizontal spaces to ignore before starting to parse", + ) + offset_row = fields.Integer( + default=0, + help="Vertical spaces to ignore before starting to parse", + ) timestamp_column = fields.Char(required=True) currency_column = fields.Char( help=( @@ -217,6 +229,12 @@ def onchange_decimal_separator(self): elif "comma" == self.float_thousands_sep == self.float_decimal_sep: self.float_thousands_sep = "dot" + @api.constrains("offset_column", "offset_row") + def _check_columns(self): + for mapping in self: + if mapping.offset_column < 0 or mapping.offset_row < 0: + raise ValidationError(_("Offsets cannot be negative")) + def _get_float_separators(self): self.ensure_one() separators = { diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py index 4697525cc..72e14cc16 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py @@ -179,7 +179,7 @@ def _parse_lines(self, mapping, data_file, currency_code): csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) header = False if not mapping.no_header: - header_line = mapping.header_lines_skip_count - 1 + header_line = mapping.offset_row + mapping.header_lines_skip_count - 1 if isinstance(csv_or_xlsx, tuple): header = [ str(value).strip() @@ -188,6 +188,8 @@ def _parse_lines(self, mapping, data_file, currency_code): else: [next(csv_or_xlsx) for _i in range(header_line)] header = [value.strip() for value in next(csv_or_xlsx)] + if mapping.offset_column: + header = header[mapping.offset_column :] # NOTE no seria necesario debit_column y credit_column ya que tenemos los # respectivos campos related @@ -213,6 +215,133 @@ def _get_values_from_column(self, values, columns, column_name): return " ".join(content_l) return content_l[0] + def _parse_row(self, mapping, currency_code, values, columns): # noqa: C901 + timestamp = self._get_values_from_column(values, columns, "timestamp_column") + currency = ( + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] + else currency_code + ) + + def _decimal(column_name): + if columns[column_name]: + return self._parse_decimal( + self._get_values_from_column(values, columns, column_name), + mapping, + ) + + amount = _decimal("amount_column") + if not amount: + amount = abs(_decimal("amount_debit_column") or 0) + if not amount: + amount = -abs(_decimal("amount_credit_column") or 0) + + balance = ( + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] + else None + ) + original_currency = ( + self._get_values_from_column(values, columns, "original_currency_column") + if columns["original_currency_column"] + else None + ) + original_amount = ( + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] + else None + ) + debit_credit = ( + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] + else None + ) + transaction_id = ( + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] + else None + ) + description = ( + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] + else None + ) + notes = ( + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] + else None + ) + reference = ( + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] + else None + ) + partner_name = ( + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] + else None + ) + bank_name = ( + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] + else None + ) + bank_account = ( + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] + else None + ) + if currency != currency_code: + return {} + + if isinstance(timestamp, str): + timestamp = datetime.strptime(timestamp, mapping.timestamp_format) + + if balance: + balance = self._parse_decimal(balance, mapping) + else: + balance = None + + if debit_credit is not None: + amount = abs(amount) + if debit_credit == mapping.debit_value: + amount = -amount + + if original_amount: + original_amount = math.copysign( + self._parse_decimal(original_amount, mapping), amount + ) + else: + original_amount = 0.0 + if mapping.amount_inverse_sign: + amount = -amount + original_amount = -original_amount + balance = -balance if balance is not None else balance + line = { + "timestamp": timestamp, + "amount": amount, + "currency": currency, + "original_amount": original_amount, + "original_currency": original_currency, + } + if balance is not None: + line["balance"] = balance + if transaction_id is not None: + line["transaction_id"] = transaction_id + if description is not None: + line["description"] = description + if notes is not None: + line["notes"] = notes + if reference is not None: + line["reference"] = reference + if partner_name is not None: + line["partner_name"] = partner_name + if bank_name is not None: + line["bank_name"] = bank_name + if bank_account is not None: + line["bank_account"] = bank_account + return line + def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 csv_or_xlsx, data_file = data @@ -222,11 +351,11 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 else: numrows = len(str(data_file.strip()).split("\\n")) - label_line = mapping.header_lines_skip_count + label_line = mapping.header_lines_skip_count + mapping.offset_row footer_line = numrows - mapping.footer_lines_skip_count if isinstance(csv_or_xlsx, tuple): - rows = range(mapping.header_lines_skip_count, footer_line) + rows = range(label_line, footer_line) else: rows = csv_or_xlsx @@ -236,7 +365,7 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(0, sheet.row_len(row)): + for col_index in range(mapping.offset_column, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: @@ -246,136 +375,11 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 if index >= footer_line: continue values = list(row) - - timestamp = self._get_values_from_column( - values, columns, "timestamp_column" - ) - currency = ( - self._get_values_from_column(values, columns, "currency_column") - if columns["currency_column"] - else currency_code - ) - - def _decimal(column_name): - if columns[column_name]: - return self._parse_decimal( - self._get_values_from_column(values, columns, column_name), - mapping, - ) - - amount = _decimal("amount_column") - if not amount: - amount = abs(_decimal("amount_debit_column") or 0) - if not amount: - amount = -abs(_decimal("amount_credit_column") or 0) - - balance = ( - self._get_values_from_column(values, columns, "balance_column") - if columns["balance_column"] - else None - ) - original_currency = ( - self._get_values_from_column( - values, columns, "original_currency_column" - ) - if columns["original_currency_column"] - else None - ) - original_amount = ( - self._get_values_from_column(values, columns, "original_amount_column") - if columns["original_amount_column"] - else None - ) - debit_credit = ( - self._get_values_from_column(values, columns, "debit_credit_column") - if columns["debit_credit_column"] - else None - ) - transaction_id = ( - self._get_values_from_column(values, columns, "transaction_id_column") - if columns["transaction_id_column"] - else None - ) - description = ( - self._get_values_from_column(values, columns, "description_column") - if columns["description_column"] - else None - ) - notes = ( - self._get_values_from_column(values, columns, "notes_column") - if columns["notes_column"] - else None - ) - reference = ( - self._get_values_from_column(values, columns, "reference_column") - if columns["reference_column"] - else None - ) - partner_name = ( - self._get_values_from_column(values, columns, "partner_name_column") - if columns["partner_name_column"] - else None - ) - bank_name = ( - self._get_values_from_column(values, columns, "bank_name_column") - if columns["bank_name_column"] - else None - ) - bank_account = ( - self._get_values_from_column(values, columns, "bank_account_column") - if columns["bank_account_column"] - else None - ) - if currency != currency_code: + if mapping.skip_empty_lines and not any(values): continue - - if isinstance(timestamp, str): - timestamp = datetime.strptime(timestamp, mapping.timestamp_format) - - if balance: - balance = self._parse_decimal(balance, mapping) - else: - balance = None - - if debit_credit is not None: - amount = abs(amount) - if debit_credit == mapping.debit_value: - amount = -amount - - if original_amount: - original_amount = math.copysign( - self._parse_decimal(original_amount, mapping), amount - ) - else: - original_amount = 0.0 - if mapping.amount_inverse_sign: - amount = -amount - original_amount = -original_amount - balance = -balance if balance is not None else balance - line = { - "timestamp": timestamp, - "amount": amount, - "currency": currency, - "original_amount": original_amount, - "original_currency": original_currency, - } - if balance is not None: - line["balance"] = balance - if transaction_id is not None: - line["transaction_id"] = transaction_id - if description is not None: - line["description"] = description - if notes is not None: - line["notes"] = notes - if reference is not None: - line["reference"] = reference - if partner_name is not None: - line["partner_name"] = partner_name - if bank_name is not None: - line["bank_name"] = bank_name - if bank_account is not None: - line["bank_account"] = bank_account - lines.append(line) + line = self._parse_row(mapping, currency_code, values, columns) + if line: + lines.append(line) return lines @api.model diff --git a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst index 36fc13e14..103db4827 100644 --- a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst +++ b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst @@ -15,3 +15,5 @@ * `CorporateHub `__ * Alexey Pelykh + +* Sebastiano Picchi diff --git a/account_statement_import_sheet_file/static/description/index.html b/account_statement_import_sheet_file/static/description/index.html index b924b868e..2b7c93f9f 100644 --- a/account_statement_import_sheet_file/static/description/index.html +++ b/account_statement_import_sheet_file/static/description/index.html @@ -8,10 +8,11 @@ /* :Author: David Goodger (goodger@python.org) -:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $ +:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $ :Copyright: This stylesheet has been placed in the public domain. Default cascading style sheet for the HTML output of Docutils. +Despite the name, some widely supported CSS2 features are used. See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to customize this style sheet. @@ -274,7 +275,7 @@ margin-left: 2em ; margin-right: 2em } -pre.code .ln { color: grey; } /* line numbers */ +pre.code .ln { color: gray; } /* line numbers */ pre.code, code { background-color: #eeeeee } pre.code .comment, code .comment { color: #5C6576 } pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold } @@ -300,7 +301,7 @@ span.pre { white-space: pre } -span.problematic { +span.problematic, pre.problematic { color: red } span.section-subtitle { @@ -455,12 +456,15 @@

Contributors

  • Alexey Pelykh <alexey.pelykh@corphub.eu>
    • +
  • Sebastiano Picchi <sebastiano.picchi@pytech.it>

    • Maintainers

    This module is maintained by the OCA.

    -Odoo Community Association + +Odoo Community Association +

    OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

    diff --git a/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv b/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv new file mode 100644 index 000000000..c5b7aded4 --- /dev/null +++ b/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv @@ -0,0 +1,5 @@ +"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account" +"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789" +"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior","" +,,,,,, +"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789" diff --git a/account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx b/account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx new file mode 100644 index 000000000..2cfc77a8a Binary files /dev/null and b/account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx differ diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py index 4c767609b..c0abcd3b1 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py @@ -1,7 +1,6 @@ # Copyright 2019 ForgeFlow, S.L. # Copyright 2020 CorporateHub (https://corporatehub.eu) # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - from base64 import b64encode from decimal import Decimal from os import path @@ -678,3 +677,95 @@ def test_decimal_and_float_inputs(self): self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot), 1234.56, ) + + def test_offsets(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + file_name = "fixtures/sample_statement_offsets.xlsx" + data = self._data_file(file_name) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + with self.assertRaises(UserError): + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement_map_offsets = self.sample_statement_map.copy( + { + "offset_column": 1, + "offset_row": 2, + } + ) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": statement_map_offsets.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 2) + self.assertEqual(statement.balance_start, 0.0) + self.assertEqual(statement.balance_end_real, 1491.5) + self.assertEqual(statement.balance_end, 1491.5) + + def test_skip_empty_lines(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + file_name = "fixtures/empty_lines_statement.csv" + data = self._data_file(file_name, "utf-8") + + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + with self.assertRaises(UserError): + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement_map_empty_line = self.sample_statement_map.copy( + { + "skip_empty_lines": True, + } + ) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": statement_map_empty_line.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 3) + self.assertEqual(statement.balance_start, 0.0) + self.assertEqual(statement.balance_end_real, 2291.5) + self.assertEqual(statement.balance_end, 2291.5) diff --git a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml index 49f34bc9e..22cee0ec4 100644 --- a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml @@ -50,6 +50,15 @@ class="fa fa-info-circle" /> indicate the column number in the Columns section. The first column is 0.
    + + +