Finished release 0.7.

.gitignore

@@ -3,4 +3,8 @@ venv
 *.pyc
 sage_validation/staticfiles
 sage_validation/frontend/node_modules
-.env
+.env
\ No newline at end of file
+docs/build
+db.sqlite3
+coverage.xml
+.coverage
\ No newline at end of file

docs/source/conf.py

@@ -12,17 +12,21 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../sage-validation'))
 
+import django
+
+sys.path.insert(0, os.path.abspath("../../"))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sage_validation.settings")
+django.setup()
 
 # -- Project information -----------------------------------------------------
 
-project = 'Sage Validation'
-copyright = '2024, GÉANT'
-author = 'GÉANT'
+project = "Sage Validation"
+copyright = "2024, GÉANT"
+author = "GÉANT"
 
 # The full version, including alpha/beta/rc tags
-release = '0.1'
+release = "0.1"
 
 
 # -- General configuration ---------------------------------------------------
@@ -31,13 +35,13 @@ release = '0.1'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.napoleon',
-    'sphinx_autodoc_typehints',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx_autodoc_typehints",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -50,9 +54,9 @@ exclude_patterns = []
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
\ No newline at end of file

docs/source/file_validator/forms.rst

+==========================================
+Sage Validation - File Validator Forms
+==========================================
+
+.. automodule:: sage_validation.file_validator.forms
+   :members:
+   :undoc-members:
+   :show-inheritance:
\ No newline at end of file

docs/source/file_validator/index.rst

+==========================================
+File Validator Module
+==========================================
+
+ .. toctree::
+   :maxdepth: 2
+   :caption: Components:
+
+   views
+   forms
\ No newline at end of file

docs/source/file_validator/views.rst

+==========================================
+Sage Validation - File Validator Views
+==========================================
+
+.. automodule:: sage_validation.file_validator.views
+   :members:
+   :undoc-members:
+   :show-inheritance:
\ No newline at end of file

docs/source/index.rst

@@ -10,11 +10,5 @@ Welcome to Sage Validation's documentation!
    :maxdepth: 2
    :caption: Contents:
 
-
-
-Indices and tables
+   validations
+   file_validator/index
\ No newline at end of file
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

docs/source/validations.rst

+====================================
+CSV Upload Validation Rules
+====================================
+
+The following table describes the validation rules applied when uploading a CSV file.
+
+.. list-table::
+   :header-rows: 1
+   :widths: 20 20 40 20
+
+   * - **Type**
+     - **Field**
+     - **Logic**
+     - **Comments**
+
+   * - Structural
+     - *(All Fields)*
+     - Ensure that all required field names are present.
+     - -
+   * - Update
+     - `...NominalAccountNumber` (*n)
+     - If value starts with `xx-`, map `NominalCostCentre` using `dbo.meoCostCentres` and retrieve `NominalAccountNumber` from `dbo.vw_xx-data`.
+     - Ensure the correct nominal code is assigned.
+
+   * - Checks
+     - AccountNumber
+     - Validate against `dbo.meoValidSuppliers`.
+     - Ensure the account exists in the valid suppliers list.
+
+   * - Checks
+     - Source
+     - Should always be `80`.
+     - Reject rows where `Source != 80`.
+
+   * - Checks
+     - SYSTraderTranType
+     - Should always be `4`.
+     - Reject rows where `SYSTraderTranType != 4`.
+
+   * - Update
+     - TransactionDate
+     - Set to today’s date (`dd/mm/yyyy`).
+     - Ensure all rows have a consistent transaction date.
+
+   * - Checks
+     - `...NominalAccountNumber` (*n)
+     - Validate against `[Nom]` in `dbo.meoNominal`.
+     - Ensure all nominal account numbers exist.
+
+   * - Checks
+     - `...NominalCostCentre` (*n)
+     - Validate against `dbo.meoCostCentres`.
+     - Ensure cost centres exist.
+
+   * - Checks
+     - Combination of `...NominalAccountNumber`, `...NominalCostCentre`, and `...NominalDepartment`
+     - Validate against `dbo.meoValidSageAccounts`.
+     - The combination of the three fields must be valid.
+
+   * - Checks
+     - `...TaxRate`
+     - Validate against `[Tax code]` in `dbo.meoValidVAT`.
+     - Ensure valid tax rates.
+
+    * - Checks
+    - `ChequeCurrencyName`, `ChequeToBankExchangeRate`, `ChequeValueInChequeCurrency`
+    - Cheque fields must be empty.
\ No newline at end of file

sage_validation/file_validator/forms.py

 """Forms for the file_validator app."""
 
 import csv
+import re
 from collections.abc import Sequence
 from typing import ClassVar
 
@@ -44,6 +45,9 @@ class CSVUploadForm(forms.Form):
         "TaxValue",
         "SYSTraderGenerationReasonType",
         "GoodsValueInBaseCurrency",
+        "ChequeCurrencyName",
+        "ChequeToBankExchangeRate",
+        "ChequeValueInChequeCurrency",
     ]
 
     repeating_columns: ClassVar[dict] = {
@@ -82,6 +86,7 @@ class CSVUploadForm(forms.Form):
         error_list.extend(self._validate_source_and_trader_type(data))
         error_list.extend(self._validate_nominal_analysis_account(data))
         error_list.extend(self._validate_nc_cc_dep_combination_against_meo_sage_account(data))
+        error_list.extend(self._cheque_fields_must_be_empty(data))
         if error_list:
             raise forms.ValidationError(error_list)
 
@@ -125,17 +130,20 @@ class CSVUploadForm(forms.Form):
             msg = f"Missing required columns: {', '.join(missing_columns)}"
             raise forms.ValidationError(msg)
 
-    @staticmethod
-    def _validate_source_and_trader_type(data: list[dict]) -> list:
+    def _validate_source_and_trader_type(self, data: list[dict]) -> list:
         """Validate that 'Source' is always 80 and 'SYSTraderTranType' is always 4."""
         errors = []
 
         for index, row in enumerate(data, start=1):
+            claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+            claim_number = row.get("SecondReference")
             if row.get("Source") != "80":
-                errors.append(f"Row {index}: 'Source' must be 80, but found {row.get('Source')}.")
+                errors.append(f"Row {index}, claimant: {claimant_name} with claim number: {claim_number}: "
+                              f"'Source' must be 80, but found {row.get('Source')}.")
 
             if row.get("SYSTraderTranType") != "4":
-                errors.append(f"Row {index}: 'SYSTraderTranType' must be 4, but found {row.get('SYSTraderTranType')}.")
+                errors.append(f"Row {index}, claimant: {claimant_name} with claim number: {claim_number}: "
+                              f"'SYSTraderTranType' must be 4, but found {row.get('SYSTraderTranType')}.")
 
         return errors
 
@@ -169,16 +177,31 @@ class CSVUploadForm(forms.Form):
                 continue
 
             pl_account_name = account_code_map.get(account_code)
+
             if pl_account_name is None:
                 errors.append(f"Row {index}: 'AccountNumber' {account_code} does not exist in PL Account Codes.")
-            elif pl_account_name not in nominal:
-                errors.append(
-                    f"Row {index}: 'AccountNumber' must match '{pl_account_name}' in "
-                    f"'NominalAnalysisNominalAnalysisNarrative/1', but found '{nominal}'."
-                )
+            else:
+                # Remove 'Soldo' and any hyphens from the PL account name. This is for credit card accounts.
+                revised_pl_account_name = re.sub(
+                    r"\bSoldo\b|\s*-\s*", "", pl_account_name, flags=re.IGNORECASE).strip()
+                if revised_pl_account_name not in nominal:
+                    errors.append(
+                        f"Row {index}: 'AccountNumber' must match '{revised_pl_account_name}' in "
+                        f"'NominalAnalysisNominalAnalysisNarrative/1', but found '{nominal}'."
+                    )
 
         return errors
 
+    @staticmethod
+    def get_account_name_from_code(account_code: str| None) -> str | None:
+        """Get the account name from the PL Account Codes table."""
+        if account_code is None:
+            return None
+        try:
+            return MeoValidSuppliers.objects.using("meo").get(
+                supplier_account_number=account_code).supplier_account_name
+        except MeoValidSuppliers.DoesNotExist:
+            return None
 
     def _validate_nc_cc_dep_combination_against_meo_sage_account(self, data: list[dict]) -> list[str]:
         """Validate that all nominal analysis fields exist in MEO.
@@ -208,6 +231,8 @@ class CSVUploadForm(forms.Form):
         max_repeat = self._get_max_repeat(fieldnames, "NominalAnalysisNominalCostCentre")
 
         for index, row in enumerate(data, start=1):
+            claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+            claim_number = row.get("SecondReference")
             for repeat in range(1, max_repeat + 1):
                 cc_field = f"NominalAnalysisNominalCostCentre/{repeat}"
                 dep_field = f"NominalAnalysisNominalDepartment/{repeat}"
@@ -241,7 +266,28 @@ class CSVUploadForm(forms.Form):
                     errors.append(
                         f"Row {index}: The combination of '{cc_field}' ({cc}), "
                         f"'{dep_field}' ({dep}), and '{nominal_account_field}' "
-                        f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
+                        f"({nc}) for claimant '{claimant_name}' and claim number '{claim_number}' "
+                        f"does not exist in MEO valid Sage accounts."
                     )
 
         return errors
+
+    def _cheque_fields_must_be_empty(self, data: list[dict]) -> list[str]:
+        """Validate that cheque fields are empty.
+
+        The cheque fields are 'ChequeCurrencyName', 'ChequeToBankExchangeRate', and 'ChequeValueInChequeCurrency'.
+        """
+        errors = []
+        for index, row in enumerate(data, start=1):
+            cheque_currency_name = row.get("ChequeCurrencyName")
+            cheque_to_bank_exchange_rate = row.get("ChequeToBankExchangeRate")
+            cheque_value_in_cheque_currency = row.get("ChequeValueInChequeCurrency")
+            claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+            claim_number = row.get("SecondReference")
+            if any([cheque_currency_name, cheque_to_bank_exchange_rate, cheque_value_in_cheque_currency]):
+                errors.append(
+                    f"Row {index}: Unexpected values in the Cheque columns for {claimant_name} with claim number: "
+                    f"{claim_number}. All cheque columns must be empty."
+                )
+
+        return errors

sage_validation/file_validator/models.py

@@ -38,6 +38,7 @@ class MeoNominal(models.Model):
 class MeoValidSageAccounts(models.Model):
     """View for MEO valid Sage accounts."""
 
+    id = models.UUIDField(db_column="ID", primary_key=True)
     account_name = models.CharField(db_column="AccountName", max_length=60)
     account_number = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
     account_cost_centre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)

sage_validation/file_validator/templates/upload.html

@@ -3,7 +3,7 @@
 {% block title %}File Upload{% endblock %}
 
 {% block content %}
-    <div class="bg-white p-10 rounded-lg shadow-lg w-11/12 md:w-1/2 lg:w-1/3 mx-auto">
+    <div class="bg-white p-10 rounded-lg shadow-lg w-11/12 md:w-3/4 lg:w-3/4 mx-auto">
         <h2 class="text-2xl font-bold mb-6 text-gray-800 text-center">Upload CSV File</h2>
 
         <form id="uploadForm" enctype="multipart/form-data" class="space-y-6">
@@ -13,10 +13,21 @@
                        class="w-full border border-gray-300 p-3 rounded-lg focus:outline-none focus:ring focus:ring-blue-400">
             </div>
 
-            <button type="submit"
+            <button id="submitBtn" type="submit"
                     class="w-full bg-blue-600 text-white py-3 rounded-lg hover:bg-blue-700 focus:outline-none focus:ring focus:ring-blue-400">
                 Upload
             </button>
+
+            <!-- Loading Spinner -->
+            <div id="loadingSpinner" class="hidden flex justify-center">
+                <svg class="animate-spin h-6 w-6 text-blue-600" xmlns="http://www.w3.org/2000/svg" fill="none"
+                     viewBox="0 0 24 24">
+                    <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor"
+                            stroke-width="4"></circle>
+                    <path class="opacity-75" fill="currentColor"
+                          d="M4 12a8 8 0 018-8v4a4 4 0 00-4 4H4z"/>
+                </svg>
+            </div>
         </form>
 
         <!-- Error Display -->
@@ -45,6 +56,8 @@
             e.preventDefault();
 
             const fileInput = document.getElementById('fileInput');
+            const submitBtn = document.getElementById('submitBtn');
+            const loadingSpinner = document.getElementById('loadingSpinner');
             const errorSection = document.getElementById('errorSection');
             const errorList = document.getElementById('errorList');
             const successSection = document.getElementById('successSection');
@@ -58,6 +71,9 @@
             successSection.classList.add('hidden');
             downloadSection.classList.add('hidden');
 
+            submitBtn.disabled = true;
+            loadingSpinner.classList.remove('hidden');
+
             const formData = new FormData();
             formData.append('file', fileInput.files[0]);
 
@@ -85,10 +101,10 @@
                         li.textContent = 'You are not authorized to perform this action.';
                         errorList.appendChild(li);
                     } else if (response.status === 400 && result.status === 'error') {
-                        for (const [field, messages] of Object.entries(result.errors)) {
+                        for (const messages of Object.values(result.errors)) {
                             messages.forEach(message => {
                                 const li = document.createElement('li');
-                                li.textContent = `${field}: ${message}`;
+                                li.textContent = message;
                                 errorList.appendChild(li);
                             });
                         }
@@ -105,6 +121,9 @@
                 li.textContent = 'Failed to connect to the server. Please check your internet connection.';
                 errorList.appendChild(li);
                 errorSection.classList.remove('hidden');
+            } finally {
+                loadingSpinner.classList.add('hidden');
+                submitBtn.disabled = false;
             }
         });
     </script>

setup.py

@@ -3,7 +3,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="sage-validation",
-    version="0.6",
+    version="0.7",
     packages=find_packages(),
     include_package_data=True,
     install_requires=[

test/conftest.py

@@ -37,6 +37,9 @@ def sample_input_file() -> SimpleUploadedFile:
         "SYSTraderGenerationReasonType",
         "GoodsValueInBaseCurrency",
         "TransactionReference",
+        "ChequeCurrencyName",
+        "ChequeToBankExchangeRate",
+        "ChequeValueInChequeCurrency",
 
         # NominalAnalysis repeating columns (Example: /1 for first occurrence)
         "NominalAnalysisTransactionValue/1",
@@ -79,6 +82,9 @@ def sample_input_file() -> SimpleUploadedFile:
         "1000",  # SYSTraderGenerationReasonType
         "1200",  # GoodsValueInBaseCurrency
         "BK123",  # TransactionReference(Batch Number)
+        "",  # ChequeCurrencyName
+        "",  # ChequeToBankExchangeRate
+        "",  # ChequeValueInChequeCurrency
 
         # NominalAnalysis repeating values (Example: /1)
         "500.75",  # NominalAnalysisTransactionValue/1

test/test_file_validator/test_forms.py

@@ -61,8 +61,8 @@ def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile
     modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"})
     form = CSVUploadForm(files={"file": modified_file})
     assert not form.is_valid()
-    assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
-    assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
+    assert "'Source' must be 80" in form.errors["file"][0]
+    assert "'SYSTraderTranType' must be 4" in form.errors["file"][1]
 
 
 def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
@@ -92,3 +92,11 @@ def test_validate_nc_cc_dep_combination_against_meo_sage_account(
     assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre."
             in str(form.errors["file"][0]))
 
+
+def test_cheque_fields_must_be_empty(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
+    """Test that cheque fields must be empty."""
+    modified_file = create_modified_csv(sample_input_file,
+                                        {"ChequeCurrencyName": "USD", "ChequeToBankExchangeRate": "1"})
+    form = CSVUploadForm(files={"file": modified_file})
+    assert not form.is_valid()
+    assert "All cheque columns must be empty." in form.errors["file"][0]