diff --git a/account_statement_import_sheet_file_ux/README.rst b/account_statement_import_sheet_file_ux/README.rst new file mode 100644 index 00000000..b6fcaca2 --- /dev/null +++ b/account_statement_import_sheet_file_ux/README.rst @@ -0,0 +1,125 @@ +.. |company| replace:: ADHOC SA + +.. |company_logo| image:: https://raw.githubusercontent.com/ingadhoc/maintainer-tools/master/resources/adhoc-logo.png + :alt: ADHOC SA + :target: https://www.adhoc.com.ar + +.. |icon| image:: https://raw.githubusercontent.com/ingadhoc/maintainer-tools/master/resources/adhoc-icon.png + +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 + +=============================== +Bank Statement Sheet Import UX +=============================== + +Most of the support around the statement sheet import comes from mappings that +do not match the file the bank exports: a renamed header, a header row on the +wrong line, the thousands separator the other way around. The error the user +gets back (``'Debits' is not in list``) says nothing about which part of the +mapping is wrong. + + * Adds a **"Preview Mapping"** button on the statement sheet mapping. It shows a + sample sheet, with column letters and row numbers, built from the mapping as + it is configured right now: where the header row must be, which columns must + exist, and how dates and amounts must be written. Three sample transactions + are filled in so the layout is unambiguous. + * The sample can be downloaded as an **xlsx** file. Every cell is written as + text, the way the parser reads them, so the sample file can be imported with + that very mapping to check it end to end before fighting with the real + statement. + * The preview reads the mapping back to the user in plain words (date format, + separators, ignored rows and columns) and **warns about the configurations + that are going to fail**: a header row number of 0 (the header row is then + read as a transaction too and the spreadsheet import fails), or column names + on a mapping declared as having no header line. + * Rewrites the error of a **failed import**. Instead of the bare + ``'Date' is not in list``, it names the mapping and the column that is + missing (or the date format that does not match), tells the user to check the + Columns section and the preview, and offers a button that opens the mapping + right there. + * Matches the configured column names **ignoring case and padding**, so a bank + that exports ``DATE`` one month and ``Date`` the next one does not break the + import. + * Renames the ``Header lines skip count`` field to **Header row number** and + explains what the number means, which is where most of the wrong mappings + come from. + * Clears the amount columns when the **Amount type** changes, so a mapping + cannot keep columns that no longer apply to it. + +Technical notes +=============== + + * The preview is built by ``account.statement.import.sheet.mapping._preview_layout()``, + which follows what the parser actually does: the header sits on + ``max(header_lines_skip_count, 1)`` and the transactions start right after + ``header_lines_skip_count``. Ignored rows carry a label so that they exist in + the exported file and the row numbers do not shift. + * With ``no_header`` the mapping holds column indexes, so each field lands on + the position it declares. With a header the file order is irrelevant to the + parser, so the preview lays the columns out in the order of the form and says + so. + * ``_get_column_indexes`` rewrites the header cells that match a configured + name to the configured spelling and delegates to the standard lookup, instead + of duplicating it. An unknown column still raises. + * ``_parse_decimal`` casts the raw ``int`` / ``float`` values the xls parser + hands over, which the standard method only handles for ``str``, ``float`` and + ``Decimal``. + * ``account.statement.import._parse_file`` catches the ``UserError`` the sheet + module raises and re-raises a ``RedirectWarning`` carrying the original text + plus the hint. The redirect action is built inline instead of pointing at a + preview record, because the failed transaction is rolled back and any record + created in that path would no longer exist when the user clicks the button. + +Installation +============ + +Only install the module. It is auto installed as soon as +``account_statement_import_sheet_file`` is installed, and it pulls +``account_statement_import_sheet_file_xls`` and +``account_statement_import_sheet_file_xlsx`` with it, so a base that can import +csv statements can also import xls and xlsx ones. + +Configuration +============= + +This module does not need any configuration. + +Usage +===== + +Go to "Accounting > Configuration > Statement Sheet Mappings", open a mapping and +click "Preview Mapping". + +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: http://runbot.adhoc.com.ar/ + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed feedback. + +Credits +======= + +Images +------ + +* |company| |icon| + +Contributors +------------ + +Maintainer +---------- + +|company_logo| + +This module is maintained by the |company|. + +To contribute to this module, please visit https://www.adhoc.com.ar. diff --git a/account_statement_import_sheet_file_ux/__init__.py b/account_statement_import_sheet_file_ux/__init__.py new file mode 100644 index 00000000..9b429614 --- /dev/null +++ b/account_statement_import_sheet_file_ux/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizard diff --git a/account_statement_import_sheet_file_ux/__manifest__.py b/account_statement_import_sheet_file_ux/__manifest__.py new file mode 100644 index 00000000..6728810a --- /dev/null +++ b/account_statement_import_sheet_file_ux/__manifest__.py @@ -0,0 +1,43 @@ +############################################################################## +# +# Copyright (C) 2026 ADHOC SA (http://www.adhoc.com.ar) +# All Rights Reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +{ + "name": "Bank Statement Sheet Import UX", + "version": "19.0.1.0.0", + "category": "Accounting", + "summary": "Preview of the expected file for a statement sheet mapping, " + "plus usability fixes on the mapping configuration and the column parser", + "author": "ADHOC SA", + "website": "www.adhoc.com.ar", + "license": "AGPL-3", + "images": [], + "depends": [ + "account_statement_import_sheet_file", + "account_statement_import_sheet_file_xls", + "account_statement_import_sheet_file_xlsx", + ], + "data": [ + "security/ir.model.access.csv", + "views/account_statement_import_sheet_mapping_preview_views.xml", + "views/account_statement_import_sheet_mapping_views.xml", + ], + "installable": True, + "auto_install": ["account_statement_import_sheet_file"], + "application": False, +} diff --git a/account_statement_import_sheet_file_ux/i18n/account_statement_import_sheet_file_ux.pot b/account_statement_import_sheet_file_ux/i18n/account_statement_import_sheet_file_ux.pot new file mode 100644 index 00000000..3571d009 --- /dev/null +++ b/account_statement_import_sheet_file_ux/i18n/account_statement_import_sheet_file_ux.pot @@ -0,0 +1,332 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * account_statement_import_sheet_file_ux +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 19.0+e\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-08-21 20:46+0000\n" +"PO-Revision-Date: 2026-08-21 20:46+0000\n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__amount_type +msgid "Amount type" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"Amounts carry no separator at all: the last %s digits are read as the " +"decimals." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"Amounts use '%(thousands)s' as thousands separator and '%(decimal)s' as " +"decimal separator." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Amounts use '%s' as decimal separator and no thousands separator." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model,name:account_statement_import_sheet_file_ux.model_account_statement_import_sheet_mapping +msgid "Bank Statement Import Sheet Mapping" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model,name:account_statement_import_sheet_file_ux.model_account_statement_import_sheet_mapping_preview +msgid "Bank Statement Import Sheet Mapping Preview" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model,name:account_statement_import_sheet_file_ux.model_account_statement_import_sheet_parser +msgid "Bank Statement Import Sheet Parser" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Bank fee" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import.py:0 +msgid "Check the mapping" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.account_statement_import_sheet_mapping_preview_form +msgid "Close" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Columns ignored at the beginning of each row: %s." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__create_uid +msgid "Created by" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__create_date +msgid "Created on" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Customer A" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Dates are read with the format %s." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import__display_name +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__display_name +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__display_name +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_parser__display_name +msgid "Display Name" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.account_statement_import_sheet_mapping_preview_form +msgid "Download Sample File" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__header_lines_skip_count +msgid "Header row number" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import__id +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__id +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__id +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_parser__id +msgid "ID" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Ignored row" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model,name:account_statement_import_sheet_file_ux.model_account_statement_import +msgid "Import Bank Statement Files" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Incoming transfer" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__write_uid +msgid "Last Updated by" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__write_date +msgid "Last Updated on" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__mapping_id +msgid "Mapping" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Mapping preview" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__preview_html +msgid "Preview" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.account_statement_import_sheet_mapping_form +msgid "Preview Mapping" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,help:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__header_lines_skip_count +msgid "" +"Row number where the column headers are located, the first row being 1. Use " +"the 'Preview Mapping' button to check how the file is expected to look with " +"the current configuration." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Rows ignored at the end of the file: %s." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Sample Bank" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__file_data +msgid "Sample File" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,field_description:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping_preview__file_name +msgid "Sample File Name" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Sample line" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.account_statement_import_sheet_mapping_form +msgid "Show how the file is expected to look with this configuration" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model:ir.model.fields,help:account_statement_import_sheet_file_ux.field_account_statement_import_sheet_mapping__amount_type +msgid "" +"Simple value: use the signed amount in the amount column\n" +"Absolute value: use a same column for debit and credit (absolute value + indicate sign)\n" +"Distinct Credit/debit Column: use a distinct column for debit and credit" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import_sheet_mapping_preview.py:0 +msgid "Statement" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import.py:0 +msgid "Statement Sheet Mapping" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Supplier B" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "Supplier payment" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import.py:0 +msgid "" +"The file could not be read with the mapping '%s'. Use the 'Preview Mapping' " +"button on the mapping to see the file it is expecting and compare it with " +"yours." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"The file has no header line, so every column has to be a number (the first " +"column is 0). These fields hold a name instead and are not shown below: %s" +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"The file must have no header row: the mapping points at column positions, so" +" the order shown here is the one your file needs." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"The header row number is 0. When importing a spreadsheet the header row is " +"then read as a transaction too and the import fails. Set it to 1 if the " +"headers are in the first row of the file." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import.py:0 +msgid "" +"The mapping '%(mapping)s' expects a column named '%(column)s' and the file " +"does not have it. Banks rename their columns from one export to the next: " +"check the Columns section of the mapping and use the 'Preview Mapping' " +"button to see the file the mapping is expecting." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/wizard/account_statement_import.py:0 +msgid "" +"The mapping '%(mapping)s' reads the dates with the format '%(format)s' and " +"the file writes them differently. Check the 'Timestamp format' field of the " +"mapping and use the 'Preview Mapping' button to see the file the mapping is " +"expecting." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"The order of the columns in your file does not matter and extra columns are " +"ignored: only the header names have to match the ones shown here." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#. odoo-python +#: code:addons/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py:0 +msgid "" +"The sample file can be imported as is: use it to check the mapping before " +"fighting with the real statement." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.mapping_preview +msgid "" +"There is no column to preview yet: fill in the Columns section of the\n" +" mapping and open this preview again." +msgstr "" + +#. module: account_statement_import_sheet_file_ux +#: model_terms:ir.ui.view,arch_db:account_statement_import_sheet_file_ux.mapping_preview +msgid "This is how your file should look like:" +msgstr "" diff --git a/account_statement_import_sheet_file_ux/models/__init__.py b/account_statement_import_sheet_file_ux/models/__init__.py new file mode 100644 index 00000000..e8fa2e2f --- /dev/null +++ b/account_statement_import_sheet_file_ux/models/__init__.py @@ -0,0 +1,2 @@ +from . import account_statement_import_sheet_mapping +from . import account_statement_import_sheet_parser diff --git a/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py new file mode 100644 index 00000000..98bd5e6c --- /dev/null +++ b/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_mapping.py @@ -0,0 +1,361 @@ +from datetime import timedelta + +from odoo import api, fields, models + +# Number of sample transaction lines shown in the preview +PREVIEW_LINES = 3 +# Signed amounts of those lines, as they are meant to end up in Odoo +PREVIEW_AMOUNTS = [1500.0, -2750.5, 350.75] +PREVIEW_OPENING_BALANCE = 10000.0 + + +class AccountStatementImportSheetMapping(models.Model): + _inherit = "account.statement.import.sheet.mapping" + + header_lines_skip_count = fields.Integer( + string="Header row number", + help="Row number where the column headers are located, the first row " + "being 1. Use the 'Preview Mapping' button to check how the file is " + "expected to look with the current configuration.", + ) + amount_type = fields.Selection( + help="Simple value: use the signed amount in the amount column\n" + "Absolute value: use a same column for debit and credit " + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct column for debit and credit", + ) + + @api.onchange("amount_type") + def _clear_amount_columns(self): + """The columns of the previous amount type no longer apply.""" + self.amount_column = False + self.debit_credit_column = False + self.amount_debit_column = False + self.amount_credit_column = False + + def action_preview_mapping(self): + """Open a sample of the file this mapping expects.""" + self.ensure_one() + preview = self.env["account.statement.import.sheet.mapping.preview"].create({"mapping_id": self.id}) + return { + "type": "ir.actions.act_window", + "name": self.env._("Mapping preview"), + "res_model": preview._name, + "res_id": preview.id, + "view_mode": "form", + "target": "new", + } + + # Preview building blocks + + def _preview_column_field_names(self): + """Mapping fields that may define a column, in the order they are laid out.""" + self.ensure_one() + names = [ + "timestamp_column", + "transaction_id_column", + "reference_column", + "partner_name_column", + "description_column", + "notes_column", + ] + if self.amount_type == "distinct_credit_debit": + names += ["amount_debit_column", "amount_credit_column"] + else: + names += ["amount_column"] + if self.amount_type == "absolute_value": + names += ["debit_credit_column"] + return names + [ + "balance_column", + "currency_column", + "original_currency_column", + "original_amount_column", + "bank_name_column", + "bank_account_column", + ] + + def _preview_column_keys(self, field_name): + """Column names (or indexes) configured on a mapping field.""" + self.ensure_one() + return [key.strip() for key in (self[field_name] or "").split(",") if key.strip()] + + def _preview_format_date(self, date): + self.ensure_one() + try: + return date.strftime(self.timestamp_format) + except (ValueError, TypeError): + return date.isoformat() + + def _preview_format_amount(self, amount): + self.ensure_one() + thousands, decimal = self._get_float_separators() + if not decimal: + # Without a decimal separator the parser shifts the value according + # to the currency decimals, so the file must hold plain digits + decimals = self.env.company.currency_id.decimal_places + return str(round(amount * 10**decimals)) + # Format with the python separators first and swap them afterwards, so + # that both can be configured to the same character without clashing + text = f"{abs(amount):,.2f}".replace(",", "\x00").replace(".", decimal) + text = text.replace("\x00", thousands) + return f"-{text}" if amount < 0 else text + + def _preview_sample_values(self): + """Return {mapping field: one already formatted value per sample line}.""" + self.ensure_one() + # The file holds the inverse sign when the mapping inverts it on import + amounts = [-amount if self.amount_inverse_sign else amount for amount in PREVIEW_AMOUNTS] + balance = PREVIEW_OPENING_BALANCE + balances = [] + for amount in amounts: + balance += amount + balances.append(balance) + today = fields.Date.context_today(self) + dates = [today - timedelta(days=days) for days in reversed(range(PREVIEW_LINES))] + currency = self.env.company.currency_id + values = { + "timestamp_column": [self._preview_format_date(date) for date in dates], + "transaction_id_column": ["000123", "000124", "000125"], + "reference_column": ["REF-0001", "REF-0002", "REF-0003"], + "partner_name_column": [ + self.env._("Customer A"), + self.env._("Supplier B"), + self.env._("Sample Bank"), + ], + "description_column": [ + self.env._("Incoming transfer"), + self.env._("Supplier payment"), + self.env._("Bank fee"), + ], + "notes_column": [self.env._("Sample line")] * PREVIEW_LINES, + "balance_column": [self._preview_format_amount(balance) for balance in balances], + "currency_column": [currency.name] * PREVIEW_LINES, + "original_currency_column": ["EUR" if currency.name == "USD" else "USD"] * PREVIEW_LINES, + "original_amount_column": [self._preview_format_amount(abs(amount) / 1000) for amount in amounts], + "bank_name_column": [self.env._("Sample Bank")] * PREVIEW_LINES, + "bank_account_column": ["0000076500000000000001"] * PREVIEW_LINES, + } + if self.amount_type == "distinct_credit_debit": + values["amount_debit_column"] = [ + self._preview_format_amount(amount) if amount > 0 else "" for amount in amounts + ] + values["amount_credit_column"] = [ + self._preview_format_amount(-amount) if amount < 0 else "" for amount in amounts + ] + elif self.amount_type == "absolute_value": + values["amount_column"] = [self._preview_format_amount(abs(amount)) for amount in amounts] + values["debit_credit_column"] = [ + self.debit_value if amount < 0 else self.credit_value for amount in amounts + ] + else: + values["amount_column"] = [self._preview_format_amount(amount) for amount in amounts] + return values + + @api.model + def _preview_split_value(self, value, count): + """Spread a sample value over the columns a mapping field concatenates.""" + if count <= 1: + return [value] + words = value.split(" ") + if len(words) < count: + return [value] + [""] * (count - 1) + return words[: count - 1] + [" ".join(words[count - 1 :])] + + @api.model + def _preview_column_letter(self, index): + """Spreadsheet letter of a 0-based column index (0 -> A, 26 -> AA).""" + letters = "" + while True: + index, remainder = divmod(index, 26) + letters = chr(ord("A") + remainder) + letters + if not index: + return letters + index -= 1 + + def _preview_columns(self): + """Return the sample columns as {0-based index: (label, [value per line])}. + + Without a header the mapping holds column indexes, so each field lands on + the position it declares. With a header the file order is irrelevant to + the parser, so the columns are laid out in the order of the form. + """ + self.ensure_one() + sample_values = self._preview_sample_values() + columns = {} + position = 0 if self.no_header else self.offset_column + for field_name in self._preview_column_field_names(): + keys = self._preview_column_keys(field_name) + if not keys: + continue + values = sample_values.get(field_name, [""] * PREVIEW_LINES) + # One list of cells per column of this field, one cell per line + cells = list( + zip( + *[self._preview_split_value(value, len(keys)) for value in values], + strict=True, + ) + ) + for key, key_cells in zip(keys, cells, strict=True): + if self.no_header: + try: + index = int(key) + except ValueError: + # Not a column index: the mapping is misconfigured, the + # preview shows it as missing instead of failing here + continue + else: + index = position + position += 1 + columns[index] = (key, list(key_cells)) + return columns + + def _preview_layout(self): + """Return the sample sheet ready to be rendered or exported. + + The layout follows what the parser actually does with the mapping: the + header sits on ``max(header_lines_skip_count, 1)`` and the transactions + start right after ``header_lines_skip_count``. + """ + self.ensure_one() + columns = self._preview_columns() + width = max(columns) + 1 if columns else 0 + header_row = None if self.no_header else max(self.header_lines_skip_count, 1) + first_data_row = self.header_lines_skip_count + 1 if self.no_header else header_row + 1 + # Ignored rows carry a label so that they exist in the exported file: + # a fully empty row would not be there and the row numbers would shift + ignored_cells = [self.env._("Ignored row")] + [""] * (width - 1) if width else [] + rows = [] + for number in range(1, first_data_row): + if number == header_row: + cells = [columns[index][0] if index in columns else "" for index in range(width)] + else: + cells = list(ignored_cells) + rows.append( + { + "number": number, + "kind": "header" if number == header_row else "ignored", + "cells": cells, + } + ) + for line in range(PREVIEW_LINES): + rows.append( + { + "number": first_data_row + line, + "kind": "data", + "cells": [columns[index][1][line] if index in columns else "" for index in range(width)], + } + ) + for footer in range(self.footer_lines_skip_count): + rows.append( + { + "number": first_data_row + PREVIEW_LINES + footer, + "kind": "ignored", + "cells": list(ignored_cells), + } + ) + return { + "letters": [self._preview_column_letter(index) for index in range(width)], + "rows": rows, + "notes": self._preview_notes(), + "warnings": self._preview_warnings(), + } + + def _preview_notes(self): + """Read this mapping back to the user in plain words.""" + self.ensure_one() + thousands, decimal = self._get_float_separators() + notes = [] + if self.no_header: + notes.append( + self.env._( + "The file must have no header row: the mapping points at " + "column positions, so the order shown here is the one your " + "file needs." + ) + ) + else: + notes.append( + self.env._( + "The order of the columns in your file does not matter and " + "extra columns are ignored: only the header names have to " + "match the ones shown here." + ) + ) + notes.append(self.env._("Dates are read with the format %s.", self.timestamp_format)) + if not decimal: + notes.append( + self.env._( + "Amounts carry no separator at all: the last %s digits are " "read as the decimals.", + self.env.company.currency_id.decimal_places, + ) + ) + elif thousands: + notes.append( + self.env._( + "Amounts use '%(thousands)s' as thousands separator and " "'%(decimal)s' as decimal separator.", + thousands=thousands, + decimal=decimal, + ) + ) + else: + notes.append( + self.env._( + "Amounts use '%s' as decimal separator and no thousands " "separator.", + decimal, + ) + ) + if self.offset_column and not self.no_header: + notes.append( + self.env._( + "Columns ignored at the beginning of each row: %s.", + self.offset_column, + ) + ) + if self.footer_lines_skip_count: + notes.append( + self.env._( + "Rows ignored at the end of the file: %s.", + self.footer_lines_skip_count, + ) + ) + notes.append( + self.env._( + "The sample file can be imported as is: use it to check the " + "mapping before fighting with the real statement." + ) + ) + return notes + + def _preview_warnings(self): + """Configuration that is going to break the import, spelled out.""" + self.ensure_one() + warnings = [] + if not self.no_header and not self.header_lines_skip_count: + warnings.append( + self.env._( + "The header row number is 0. When importing a spreadsheet " + "the header row is then read as a transaction too and the " + "import fails. Set it to 1 if the headers are in the first " + "row of the file." + ) + ) + if self.no_header: + missing = [ + field_name + for field_name in self._preview_column_field_names() + for key in self._preview_column_keys(field_name) + if not key.lstrip("+-").isdigit() + ] + if missing: + warnings.append( + self.env._( + "The file has no header line, so every column has to be " + "a number (the first column is 0). These fields hold a " + "name instead and are not shown below: %s", + ", ".join( + self._fields[field_name].get_description(self.env)["string"] + for field_name in dict.fromkeys(missing) + ), + ) + ) + return warnings diff --git a/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_parser.py new file mode 100644 index 00000000..fef1c348 --- /dev/null +++ b/account_statement_import_sheet_file_ux/models/account_statement_import_sheet_parser.py @@ -0,0 +1,31 @@ +from decimal import Decimal + +from odoo import api, models + + +class AccountStatementImportSheetParser(models.TransientModel): + _inherit = "account.statement.import.sheet.parser" + + def _get_column_indexes(self, header, column_name, mapping): + """Match the configured column names ignoring case and padding. + + Banks change the capitalization of their headers between exports and the + whole import fails on a ``'Debit' is not in list`` error. Rewrite the + header cells that match a configured name to the configured spelling so + the standard lookup finds them. + """ + try: + return super()._get_column_indexes(header, column_name, mapping) + except ValueError: + configured = { + key.strip().lower(): key.strip() for key in (mapping[column_name] or "").split(",") if key.strip() + } + normalized = [configured.get(str(value).strip().lower(), value) for value in header] + return super()._get_column_indexes(normalized, column_name, mapping) + + @api.model + def _parse_decimal(self, value, mapping): + """Accept the raw numbers the xls parser hands over, not only strings.""" + if not isinstance(value, Decimal | float): + value = str(value) + return super()._parse_decimal(value, mapping) diff --git a/account_statement_import_sheet_file_ux/security/ir.model.access.csv b/account_statement_import_sheet_file_ux/security/ir.model.access.csv new file mode 100644 index 00000000..9443826f --- /dev/null +++ b/account_statement_import_sheet_file_ux/security/ir.model.access.csv @@ -0,0 +1,2 @@ +"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" +access_account_statement_import_sheet_mapping_preview,account.statement.import.sheet.mapping.preview:account.group_account_user,model_account_statement_import_sheet_mapping_preview,account.group_account_user,1,1,1,1 diff --git a/account_statement_import_sheet_file_ux/tests/__init__.py b/account_statement_import_sheet_file_ux/tests/__init__.py new file mode 100644 index 00000000..09a844ed --- /dev/null +++ b/account_statement_import_sheet_file_ux/tests/__init__.py @@ -0,0 +1 @@ +from . import test_mapping_preview diff --git a/account_statement_import_sheet_file_ux/tests/test_mapping_preview.py b/account_statement_import_sheet_file_ux/tests/test_mapping_preview.py new file mode 100644 index 00000000..46945c46 --- /dev/null +++ b/account_statement_import_sheet_file_ux/tests/test_mapping_preview.py @@ -0,0 +1,242 @@ +from odoo.addons.account_statement_import_sheet_file_ux.models import ( + account_statement_import_sheet_mapping as mapping_module, +) +from odoo.exceptions import RedirectWarning +from odoo.tests.common import TransactionCase + + +class TestMappingPreview(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.parser = cls.env["account.statement.import.sheet.parser"] + cls.Mapping = cls.env["account.statement.import.sheet.mapping"] + cls.Preview = cls.env["account.statement.import.sheet.mapping.preview"] + cls.mapping_values = { + "name": "Test Mapping", + "timestamp_format": "%d/%m/%Y", + "timestamp_column": "Date", + "description_column": "Label", + "amount_type": "simple_value", + "amount_column": "Amount", + "balance_column": "Balance", + "float_thousands_sep": "dot", + "float_decimal_sep": "comma", + "header_lines_skip_count": 1, + } + cls.mapping = cls.Mapping.create(dict(cls.mapping_values)) + cls.journal = cls.env["account.journal"].create( + { + "name": "Bank UX", + "type": "bank", + "code": "BNKUX", + "currency_id": cls.env.company.currency_id.id, + } + ) + + def _layout(self, **values): + mapping = self.Mapping.create(dict(self.mapping_values, **values)) + return mapping, mapping._preview_layout() + + def _rows(self, layout, kind): + return [row for row in layout["rows"] if row["kind"] == kind] + + # Layout + + def test_header_row_follows_the_configured_row_number(self): + """The header lands where the parser looks for it, not one row off.""" + for row_number, expected in [(0, 1), (1, 1), (2, 2), (5, 5)]: + with self.subTest(header_lines_skip_count=row_number): + _mapping, layout = self._layout(header_lines_skip_count=row_number) + header_rows = self._rows(layout, "header") + self.assertEqual(len(header_rows), 1) + self.assertEqual(header_rows[0]["number"], expected) + self.assertEqual(self._rows(layout, "data")[0]["number"], expected + 1) + + def test_header_row_zero_is_warned_about(self): + """0 makes the spreadsheet parser read the header as a transaction.""" + _mapping, layout = self._layout(header_lines_skip_count=0) + self.assertTrue(layout["warnings"]) + _mapping, layout = self._layout(header_lines_skip_count=1) + self.assertFalse(layout["warnings"]) + + def test_columns_are_offset_and_footer_rows_are_ignored(self): + _mapping, layout = self._layout(offset_column=2, footer_lines_skip_count=2) + self.assertEqual(layout["letters"][:3], ["A", "B", "C"]) + header = self._rows(layout, "header")[0] + self.assertEqual(header["cells"][:3], ["", "", "Date"]) + # two leading ignored rows are not there, but the two footer ones are + self.assertEqual(len(self._rows(layout, "ignored")), 2) + + def test_sample_lines_are_the_configured_amount_type(self): + _mapping, layout = self._layout() + amounts = [row["cells"][2] for row in self._rows(layout, "data")] + self.assertEqual(amounts, ["1.500,00", "-2.750,50", "350,75"]) + _mapping, layout = self._layout( + amount_type="absolute_value", + debit_credit_column="D/C", + debit_value="D", + credit_value="C", + ) + data = self._rows(layout, "data") + self.assertEqual([row["cells"][2] for row in data], ["1.500,00", "2.750,50", "350,75"]) + self.assertEqual([row["cells"][3] for row in data], ["C", "D", "C"]) + _mapping, layout = self._layout( + amount_type="distinct_credit_debit", + amount_column=False, + amount_debit_column="Debit", + amount_credit_column="Credit", + ) + data = self._rows(layout, "data") + self.assertEqual([row["cells"][2] for row in data], ["1.500,00", "", "350,75"]) + self.assertEqual([row["cells"][3] for row in data], ["", "2.750,50", ""]) + + def test_no_header_places_columns_on_their_index(self): + _mapping, layout = self._layout( + no_header=True, + timestamp_column="0", + description_column="2", + amount_column="3", + balance_column="5", + ) + self.assertEqual(len(layout["letters"]), 6) + first = self._rows(layout, "data")[0] + self.assertEqual(first["cells"][1], "") + self.assertEqual(first["cells"][3], "1.500,00") + self.assertFalse(self._rows(layout, "header")) + + def test_no_header_with_column_names_is_warned_about(self): + _mapping, layout = self._layout(no_header=True) + self.assertTrue(layout["warnings"]) + self.assertFalse(layout["letters"]) + + def test_concatenated_columns_get_one_column_each(self): + _mapping, layout = self._layout(description_column="Label,Detail") + header = self._rows(layout, "header")[0] + self.assertEqual(header["cells"][1:3], ["Label", "Detail"]) + # the sample value is spread over both columns, not repeated + first = self._rows(layout, "data")[0] + self.assertTrue(first["cells"][1]) + self.assertTrue(first["cells"][2]) + + def test_amounts_without_decimal_separator_are_shifted(self): + _mapping, layout = self._layout(float_decimal_sep="none") + amounts = [row["cells"][2] for row in self._rows(layout, "data")] + self.assertEqual(amounts, ["150000", "-275050", "35075"]) + + def test_inverse_sign_is_reflected_in_the_file(self): + _mapping, layout = self._layout(amount_inverse_sign=True) + amounts = [row["cells"][2] for row in self._rows(layout, "data")] + self.assertEqual(amounts, ["-1.500,00", "2.750,50", "-350,75"]) + + # Wizard + + def test_preview_renders_a_table_and_an_xlsx(self): + preview = self.Preview.create({"mapping_id": self.mapping.id}) + self.assertIn(" + + + + account.statement.import.sheet.mapping.preview.form + account.statement.import.sheet.mapping.preview + +
+ + +
+
+ + +
+ diff --git a/account_statement_import_sheet_file_ux/views/account_statement_import_sheet_mapping_views.xml b/account_statement_import_sheet_file_ux/views/account_statement_import_sheet_mapping_views.xml new file mode 100644 index 00000000..50044f28 --- /dev/null +++ b/account_statement_import_sheet_file_ux/views/account_statement_import_sheet_mapping_views.xml @@ -0,0 +1,25 @@ + + + + account.statement.import.sheet.mapping.form.ux + account.statement.import.sheet.mapping + + + + +
+
+ + + + diff --git a/account_statement_import_sheet_file_ux/wizard/__init__.py b/account_statement_import_sheet_file_ux/wizard/__init__.py new file mode 100644 index 00000000..a81b5e58 --- /dev/null +++ b/account_statement_import_sheet_file_ux/wizard/__init__.py @@ -0,0 +1,2 @@ +from . import account_statement_import +from . import account_statement_import_sheet_mapping_preview diff --git a/account_statement_import_sheet_file_ux/wizard/account_statement_import.py b/account_statement_import_sheet_file_ux/wizard/account_statement_import.py new file mode 100644 index 00000000..b39054cf --- /dev/null +++ b/account_statement_import_sheet_file_ux/wizard/account_statement_import.py @@ -0,0 +1,64 @@ +import re + +from odoo import models +from odoo.exceptions import RedirectWarning, UserError + + +class AccountStatementImport(models.TransientModel): + _inherit = "account.statement.import" + + def _parse_file(self, data_file): + """Explain a failed sheet import instead of leaking the parser error. + + The standard message ("'Date' is not in list") says nothing about what + to do next, and the answer is almost always the same: the mapping does + not describe the file the bank exported. + """ + try: + return super()._parse_file(data_file) + except UserError as error: + mapping = self.sheet_mapping_id + if not mapping: + raise + raise RedirectWarning( + f"{error}\n\n{self._sheet_mapping_error_hint(mapping, str(error))}", + { + "type": "ir.actions.act_window", + "name": self.env._("Statement Sheet Mapping"), + "res_model": mapping._name, + "res_id": mapping.id, + "views": [(False, "form")], + "target": "current", + }, + self.env._("Check the mapping"), + ) from error + + def _sheet_mapping_error_hint(self, mapping, message): + """Turn a raw parser error into something the user can act on.""" + self.ensure_one() + missing_column = re.search(r"'(.+?)' is not in list", message) + if missing_column: + return self.env._( + "The mapping '%(mapping)s' expects a column named " + "'%(column)s' and the file does not have it. Banks rename " + "their columns from one export to the next: check the Columns " + "section of the mapping and use the 'Preview Mapping' button " + "to see the file the mapping is expecting.", + mapping=mapping.name, + column=missing_column.group(1), + ) + if "does not match format" in message: + return self.env._( + "The mapping '%(mapping)s' reads the dates with the format " + "'%(format)s' and the file writes them differently. Check the " + "'Timestamp format' field of the mapping and use the 'Preview " + "Mapping' button to see the file the mapping is expecting.", + mapping=mapping.name, + format=mapping.timestamp_format, + ) + return self.env._( + "The file could not be read with the mapping '%s'. Use the " + "'Preview Mapping' button on the mapping to see the file it is " + "expecting and compare it with yours.", + mapping.name, + ) diff --git a/account_statement_import_sheet_file_ux/wizard/account_statement_import_sheet_mapping_preview.py b/account_statement_import_sheet_file_ux/wizard/account_statement_import_sheet_mapping_preview.py new file mode 100644 index 00000000..1e4affbf --- /dev/null +++ b/account_statement_import_sheet_file_ux/wizard/account_statement_import_sheet_mapping_preview.py @@ -0,0 +1,84 @@ +import base64 +import io +from urllib.parse import urlencode + +import xlsxwriter +from odoo import api, fields, models + + +class AccountStatementImportSheetMappingPreview(models.TransientModel): + _name = "account.statement.import.sheet.mapping.preview" + _description = "Bank Statement Import Sheet Mapping Preview" + + mapping_id = fields.Many2one( + comodel_name="account.statement.import.sheet.mapping", + string="Mapping", + required=True, + readonly=True, + ondelete="cascade", + ) + preview_html = fields.Html( + string="Preview", + compute="_compute_preview_html", + sanitize=False, + ) + file_data = fields.Binary(string="Sample File", readonly=True, attachment=False) + file_name = fields.Char(string="Sample File Name", readonly=True) + + @api.depends("mapping_id") + def _compute_preview_html(self): + for preview in self: + preview.preview_html = self.env["ir.qweb"]._render( + "account_statement_import_sheet_file_ux.mapping_preview", + preview.mapping_id._preview_layout(), + ) + + def _build_xlsx(self): + """Return the sample sheet as xlsx bytes. + + Every cell is written as text on purpose: that is how the parser reads + the file, so the sample can be imported with this very mapping. + """ + self.ensure_one() + layout = self.mapping_id._preview_layout() + output = io.BytesIO() + workbook = xlsxwriter.Workbook(output, {"in_memory": True}) + sheet = workbook.add_worksheet(self.env._("Statement")) + header_format = workbook.add_format({"bold": True, "border": 1}) + for index in range(len(layout["letters"])): + sheet.set_column(index, index, 18) + for row in layout["rows"]: + for index, value in enumerate(row["cells"]): + if not value: + continue + sheet.write_string( + row["number"] - 1, + index, + value, + header_format if row["kind"] == "header" else None, + ) + workbook.close() + return output.getvalue() + + def action_download_xlsx(self): + self.ensure_one() + self.write( + { + "file_data": base64.b64encode(self._build_xlsx()), + "file_name": f"{self.mapping_id.name}.xlsx", + } + ) + params = urlencode( + { + "model": self._name, + "id": self.id, + "field": "file_data", + "filename": self.file_name, + "download": "true", + } + ) + return { + "type": "ir.actions.act_url", + "url": f"/web/content/?{params}", + "target": "self", + }