Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[16.0] [FW-PORT] account_statement_import_sheet_file: additional mapping controls #740

Open
wants to merge 2 commits into
base: 16.0
Choose a base branch
from
Open

[16.0] [FW-PORT] account_statement_import_sheet_file: additional mapping controls #740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1c15cd
[IMP] account_statement_import_txt_xlsx - improve column inheratibility
jdidderen-noviat Jun 21, 2023
14fe934
[IMP]account_statement_import_txt_xlsx: additional mapping controls
PicchiSeba Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions account_statement_import_sheet_file/README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ Contributors

* Alexey Pelykh <[email protected]>

* Sebastiano Picchi <[email protected]>

Maintainers
~~~~~~~~~~~

Expand Down
18 changes: 18 additions & 0 deletions account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,18 @@
help="When this occurs please indicate the column number in the Columns section "
"instead of the column name, considering that the first column is 0",
)
skip_empty_lines = fields.Boolean(
default=False,
help="Allows to skip empty lines",
)
offset_column = fields.Integer(
default=0,
help="Horizontal spaces to ignore before starting to parse",
)
offset_row = fields.Integer(
default=0,
help="Vertical spaces to ignore before starting to parse",
)
timestamp_column = fields.Char(required=True)
currency_column = fields.Char(
help=(
Expand Down Expand Up @@ -217,6 +229,12 @@
elif "comma" == self.float_thousands_sep == self.float_decimal_sep:
self.float_thousands_sep = "dot"

@api.constrains("offset_column", "offset_row")
def _check_columns(self):
for mapping in self:
if mapping.offset_column < 0 or mapping.offset_row < 0:
raise ValidationError(_("Offsets cannot be negative"))

Check warning on line 236 in account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py

View check run for this annotation

Codecov / codecov/patch

account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py#L236 

Added line #L236 was not covered by tests

def _get_float_separators(self):
self.ensure_one()
separators = {
Expand Down
270 changes: 137 additions & 133 deletions account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@
csv_or_xlsx = reader(StringIO(decoded_file), **csv_options)
header = False
if not mapping.no_header:
renda-dev marked this conversation as resolved.
Show resolved Hide resolved
header_line = mapping.header_lines_skip_count - 1
header_line = mapping.offset_row + mapping.header_lines_skip_count - 1
if isinstance(csv_or_xlsx, tuple):
header = [
str(value).strip()
Expand All @@ -188,6 +188,8 @@
else:
[next(csv_or_xlsx) for _i in range(header_line)]
header = [value.strip() for value in next(csv_or_xlsx)]
if mapping.offset_column:
header = header[mapping.offset_column :]

# NOTE no seria necesario debit_column y credit_column ya que tenemos los
# respectivos campos related
Expand All @@ -213,6 +215,133 @@
return " ".join(content_l)
return content_l[0]

def _parse_row(self, mapping, currency_code, values, columns): # noqa: C901
timestamp = self._get_values_from_column(values, columns, "timestamp_column")
currency = (
self._get_values_from_column(values, columns, "currency_column")
if columns["currency_column"]
else currency_code
)

def _decimal(column_name):
if columns[column_name]:
return self._parse_decimal(
self._get_values_from_column(values, columns, column_name),
mapping,
)

amount = _decimal("amount_column")
if not amount:
amount = abs(_decimal("amount_debit_column") or 0)
if not amount:
amount = -abs(_decimal("amount_credit_column") or 0)

balance = (
self._get_values_from_column(values, columns, "balance_column")
if columns["balance_column"]
else None
)
original_currency = (
self._get_values_from_column(values, columns, "original_currency_column")
if columns["original_currency_column"]
else None
)
original_amount = (
self._get_values_from_column(values, columns, "original_amount_column")
if columns["original_amount_column"]
else None
)
debit_credit = (
self._get_values_from_column(values, columns, "debit_credit_column")
if columns["debit_credit_column"]
else None
)
transaction_id = (
self._get_values_from_column(values, columns, "transaction_id_column")
if columns["transaction_id_column"]
else None
)
description = (
self._get_values_from_column(values, columns, "description_column")
if columns["description_column"]
else None
)
notes = (
self._get_values_from_column(values, columns, "notes_column")
if columns["notes_column"]
else None
)
reference = (
self._get_values_from_column(values, columns, "reference_column")
if columns["reference_column"]
else None
)
partner_name = (
self._get_values_from_column(values, columns, "partner_name_column")
if columns["partner_name_column"]
else None
)
bank_name = (
self._get_values_from_column(values, columns, "bank_name_column")
if columns["bank_name_column"]
else None
)
bank_account = (
self._get_values_from_column(values, columns, "bank_account_column")
if columns["bank_account_column"]
else None
)
if currency != currency_code:
return {}

if isinstance(timestamp, str):
timestamp = datetime.strptime(timestamp, mapping.timestamp_format)

if balance:
balance = self._parse_decimal(balance, mapping)
else:
balance = None

if debit_credit is not None:
amount = abs(amount)
if debit_credit == mapping.debit_value:
amount = -amount

if original_amount:
original_amount = math.copysign(
self._parse_decimal(original_amount, mapping), amount
)
else:
original_amount = 0.0
if mapping.amount_inverse_sign:
amount = -amount
original_amount = -original_amount
balance = -balance if balance is not None else balance
line = {
"timestamp": timestamp,
"amount": amount,
"currency": currency,
"original_amount": original_amount,
"original_currency": original_currency,
}
if balance is not None:
line["balance"] = balance
if transaction_id is not None:
line["transaction_id"] = transaction_id

Check warning on line 330 in account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

View check run for this annotation

Codecov / codecov/patch

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py#L330 

Added line #L330 was not covered by tests
if description is not None:
line["description"] = description
if notes is not None:
line["notes"] = notes

Check warning on line 334 in account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

View check run for this annotation

Codecov / codecov/patch

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py#L334 

Added line #L334 was not covered by tests
if reference is not None:
line["reference"] = reference
if partner_name is not None:
line["partner_name"] = partner_name
if bank_name is not None:
line["bank_name"] = bank_name

Check warning on line 340 in account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

View check run for this annotation

Codecov / codecov/patch

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py#L340 

Added line #L340 was not covered by tests
if bank_account is not None:
line["bank_account"] = bank_account
return line

def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901
csv_or_xlsx, data_file = data

Expand All @@ -222,11 +351,11 @@
else:
numrows = len(str(data_file.strip()).split("\\n"))

label_line = mapping.header_lines_skip_count
label_line = mapping.header_lines_skip_count + mapping.offset_row
footer_line = numrows - mapping.footer_lines_skip_count

if isinstance(csv_or_xlsx, tuple):
rows = range(mapping.header_lines_skip_count, footer_line)
rows = range(label_line, footer_line)
else:
rows = csv_or_xlsx

Expand All @@ -236,7 +365,7 @@
book = csv_or_xlsx[0]
sheet = csv_or_xlsx[1]
values = []
for col_index in range(0, sheet.row_len(row)):
for col_index in range(mapping.offset_column, sheet.row_len(row)):
cell_type = sheet.cell_type(row, col_index)
cell_value = sheet.cell_value(row, col_index)
if cell_type == xlrd.XL_CELL_DATE:
Expand All @@ -246,136 +375,11 @@
if index >= footer_line:
continue
values = list(row)

timestamp = self._get_values_from_column(
values, columns, "timestamp_column"
)
currency = (
self._get_values_from_column(values, columns, "currency_column")
if columns["currency_column"]
else currency_code
)

def _decimal(column_name):
if columns[column_name]:
return self._parse_decimal(
self._get_values_from_column(values, columns, column_name),
mapping,
)

amount = _decimal("amount_column")
if not amount:
amount = abs(_decimal("amount_debit_column") or 0)
if not amount:
amount = -abs(_decimal("amount_credit_column") or 0)

balance = (
self._get_values_from_column(values, columns, "balance_column")
if columns["balance_column"]
else None
)
original_currency = (
self._get_values_from_column(
values, columns, "original_currency_column"
)
if columns["original_currency_column"]
else None
)
original_amount = (
self._get_values_from_column(values, columns, "original_amount_column")
if columns["original_amount_column"]
else None
)
debit_credit = (
self._get_values_from_column(values, columns, "debit_credit_column")
if columns["debit_credit_column"]
else None
)
transaction_id = (
self._get_values_from_column(values, columns, "transaction_id_column")
if columns["transaction_id_column"]
else None
)
description = (
self._get_values_from_column(values, columns, "description_column")
if columns["description_column"]
else None
)
notes = (
self._get_values_from_column(values, columns, "notes_column")
if columns["notes_column"]
else None
)
reference = (
self._get_values_from_column(values, columns, "reference_column")
if columns["reference_column"]
else None
)
partner_name = (
self._get_values_from_column(values, columns, "partner_name_column")
if columns["partner_name_column"]
else None
)
bank_name = (
self._get_values_from_column(values, columns, "bank_name_column")
if columns["bank_name_column"]
else None
)
bank_account = (
self._get_values_from_column(values, columns, "bank_account_column")
if columns["bank_account_column"]
else None
)
if currency != currency_code:
if mapping.skip_empty_lines and not any(values):
continue

if isinstance(timestamp, str):
timestamp = datetime.strptime(timestamp, mapping.timestamp_format)

if balance:
balance = self._parse_decimal(balance, mapping)
else:
balance = None

if debit_credit is not None:
amount = abs(amount)
if debit_credit == mapping.debit_value:
amount = -amount

if original_amount:
original_amount = math.copysign(
self._parse_decimal(original_amount, mapping), amount
)
else:
original_amount = 0.0
if mapping.amount_inverse_sign:
amount = -amount
original_amount = -original_amount
balance = -balance if balance is not None else balance
line = {
"timestamp": timestamp,
"amount": amount,
"currency": currency,
"original_amount": original_amount,
"original_currency": original_currency,
}
if balance is not None:
line["balance"] = balance
if transaction_id is not None:
line["transaction_id"] = transaction_id
if description is not None:
line["description"] = description
if notes is not None:
line["notes"] = notes
if reference is not None:
line["reference"] = reference
if partner_name is not None:
line["partner_name"] = partner_name
if bank_name is not None:
line["bank_name"] = bank_name
if bank_account is not None:
line["bank_account"] = bank_account
lines.append(line)
line = self._parse_row(mapping, currency_code, values, columns)
if line:
lines.append(line)
return lines

@api.model
Expand Down
2 changes: 2 additions & 0 deletions account_statement_import_sheet_file/readme/CONTRIBUTORS.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,5 @@
* `CorporateHub <https://corporatehub.eu/>`__

* Alexey Pelykh <[email protected]>

* Sebastiano Picchi <[email protected]>
Loading
Loading