diff --git a/.gitignore b/.gitignore
index 7ea6ec68ec2c5a6b4e552fddc0405ec2159ddadf..d5c686e955ffb227ad37e6fc7d239434469bf341 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,8 @@ venv
*.pyc
sage_validation/staticfiles
sage_validation/frontend/node_modules
-.env
\ No newline at end of file
+.env
+docs/build
+db.sqlite3
+coverage.xml
+.coverage
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5cb62b06f7730cd2c493e068b4f21a68c1df7e37..38af583cffad5bb6cc39c40f8c68f91a06da1544 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -12,17 +12,21 @@
#
import os
import sys
-sys.path.insert(0, os.path.abspath('../sage-validation'))
+import django
+
+sys.path.insert(0, os.path.abspath("../../"))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sage_validation.settings")
+django.setup()
# -- Project information -----------------------------------------------------
-project = 'Sage Validation'
-copyright = '2024, GÉANT'
-author = 'GÉANT'
+project = "Sage Validation"
+copyright = "2024, GÉANT"
+author = "GÉANT"
# The full version, including alpha/beta/rc tags
-release = '0.1'
+release = "0.1"
# -- General configuration ---------------------------------------------------
@@ -31,13 +35,13 @@ release = '0.1'
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx.ext.napoleon',
- 'sphinx_autodoc_typehints',
+ "sphinx.ext.autodoc",
+ "sphinx.ext.napoleon",
+ "sphinx_autodoc_typehints",
]
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
@@ -50,9 +54,9 @@ exclude_patterns = []
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = 'alabaster'
+html_theme = "alabaster"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
\ No newline at end of file
+html_static_path = ["_static"]
diff --git a/docs/source/file_validator/forms.rst b/docs/source/file_validator/forms.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4898ae2f3027a9df4686d04c3a4748af40d0d940
--- /dev/null
+++ b/docs/source/file_validator/forms.rst
@@ -0,0 +1,8 @@
+==========================================
+Sage Validation - File Validator Forms
+==========================================
+
+.. automodule:: sage_validation.file_validator.forms
+ :members:
+ :undoc-members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/docs/source/file_validator/index.rst b/docs/source/file_validator/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..2d169ebfccd7efcc772a84ad29b43ef165804cb6
--- /dev/null
+++ b/docs/source/file_validator/index.rst
@@ -0,0 +1,10 @@
+==========================================
+File Validator Module
+==========================================
+
+ .. toctree::
+ :maxdepth: 2
+ :caption: Components:
+
+ views
+ forms
\ No newline at end of file
diff --git a/docs/source/file_validator/views.rst b/docs/source/file_validator/views.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5d27fc39a43085f0b3b4802668b07c0a73613b27
--- /dev/null
+++ b/docs/source/file_validator/views.rst
@@ -0,0 +1,8 @@
+==========================================
+Sage Validation - File Validator Views
+==========================================
+
+.. automodule:: sage_validation.file_validator.views
+ :members:
+ :undoc-members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 2af1b6c9e73ac03dce2472f8a752e770c08ea653..a050f1caa539762e6b3e00d3a3a887b0c9efdc0d 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -10,11 +10,5 @@ Welcome to Sage Validation's documentation!
:maxdepth: 2
:caption: Contents:
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+ validations
+ file_validator/index
\ No newline at end of file
diff --git a/docs/source/validations.rst b/docs/source/validations.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5f2042ced17773463685c99c6ca99a4beaafc69b
--- /dev/null
+++ b/docs/source/validations.rst
@@ -0,0 +1,67 @@
+====================================
+CSV Upload Validation Rules
+====================================
+
+The following table describes the validation rules applied when uploading a CSV file.
+
+.. list-table::
+ :header-rows: 1
+ :widths: 20 20 40 20
+
+ * - **Type**
+ - **Field**
+ - **Logic**
+ - **Comments**
+
+ * - Structural
+ - *(All Fields)*
+ - Ensure that all required field names are present.
+ - -
+ * - Update
+ - `...NominalAccountNumber` (*n)
+ - If value starts with `xx-`, map `NominalCostCentre` using `dbo.meoCostCentres` and retrieve `NominalAccountNumber` from `dbo.vw_xx-data`.
+ - Ensure the correct nominal code is assigned.
+
+ * - Checks
+ - AccountNumber
+ - Validate against `dbo.meoValidSuppliers`.
+ - Ensure the account exists in the valid suppliers list.
+
+ * - Checks
+ - Source
+ - Should always be `80`.
+ - Reject rows where `Source != 80`.
+
+ * - Checks
+ - SYSTraderTranType
+ - Should always be `4`.
+ - Reject rows where `SYSTraderTranType != 4`.
+
+ * - Update
+ - TransactionDate
+ - Set to today’s date (`dd/mm/yyyy`).
+ - Ensure all rows have a consistent transaction date.
+
+ * - Checks
+ - `...NominalAccountNumber` (*n)
+ - Validate against `[Nom]` in `dbo.meoNominal`.
+ - Ensure all nominal account numbers exist.
+
+ * - Checks
+ - `...NominalCostCentre` (*n)
+ - Validate against `dbo.meoCostCentres`.
+ - Ensure cost centres exist.
+
+ * - Checks
+ - Combination of `...NominalAccountNumber`, `...NominalCostCentre`, and `...NominalDepartment`
+ - Validate against `dbo.meoValidSageAccounts`.
+ - The combination of the three fields must be valid.
+
+ * - Checks
+ - `...TaxRate`
+ - Validate against `[Tax code]` in `dbo.meoValidVAT`.
+ - Ensure valid tax rates.
+
+ * - Checks
+ - `ChequeCurrencyName`, `ChequeToBankExchangeRate`, `ChequeValueInChequeCurrency`
+ - Cheque fields must be empty.
\ No newline at end of file
diff --git a/sage_validation/file_validator/forms.py b/sage_validation/file_validator/forms.py
index 6c243d2efe673bb8a4fe661a1852dfa6bb7cf776..ef6e9ec49fccd201adcc9a8ec30a53fbfee2e566 100644
--- a/sage_validation/file_validator/forms.py
+++ b/sage_validation/file_validator/forms.py
@@ -1,6 +1,7 @@
"""Forms for the file_validator app."""
import csv
+import re
from collections.abc import Sequence
from typing import ClassVar
@@ -44,6 +45,9 @@ class CSVUploadForm(forms.Form):
"TaxValue",
"SYSTraderGenerationReasonType",
"GoodsValueInBaseCurrency",
+ "ChequeCurrencyName",
+ "ChequeToBankExchangeRate",
+ "ChequeValueInChequeCurrency",
]
repeating_columns: ClassVar[dict] = {
@@ -82,6 +86,7 @@ class CSVUploadForm(forms.Form):
error_list.extend(self._validate_source_and_trader_type(data))
error_list.extend(self._validate_nominal_analysis_account(data))
error_list.extend(self._validate_nc_cc_dep_combination_against_meo_sage_account(data))
+ error_list.extend(self._cheque_fields_must_be_empty(data))
if error_list:
raise forms.ValidationError(error_list)
@@ -125,17 +130,20 @@ class CSVUploadForm(forms.Form):
msg = f"Missing required columns: {', '.join(missing_columns)}"
raise forms.ValidationError(msg)
- @staticmethod
- def _validate_source_and_trader_type(data: list[dict]) -> list:
+ def _validate_source_and_trader_type(self, data: list[dict]) -> list:
"""Validate that 'Source' is always 80 and 'SYSTraderTranType' is always 4."""
errors = []
for index, row in enumerate(data, start=1):
+ claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+ claim_number = row.get("SecondReference")
if row.get("Source") != "80":
- errors.append(f"Row {index}: 'Source' must be 80, but found {row.get('Source')}.")
+ errors.append(f"Row {index}, claimant: {claimant_name} with claim number: {claim_number}: "
+ f"'Source' must be 80, but found {row.get('Source')}.")
if row.get("SYSTraderTranType") != "4":
- errors.append(f"Row {index}: 'SYSTraderTranType' must be 4, but found {row.get('SYSTraderTranType')}.")
+ errors.append(f"Row {index}, claimant: {claimant_name} with claim number: {claim_number}: "
+ f"'SYSTraderTranType' must be 4, but found {row.get('SYSTraderTranType')}.")
return errors
@@ -169,16 +177,31 @@ class CSVUploadForm(forms.Form):
continue
pl_account_name = account_code_map.get(account_code)
+
if pl_account_name is None:
errors.append(f"Row {index}: 'AccountNumber' {account_code} does not exist in PL Account Codes.")
- elif pl_account_name not in nominal:
- errors.append(
- f"Row {index}: 'AccountNumber' must match '{pl_account_name}' in "
- f"'NominalAnalysisNominalAnalysisNarrative/1', but found '{nominal}'."
- )
+ else:
+ # Remove 'Soldo' and any hyphens from the PL account name. This is for credit card accounts.
+ revised_pl_account_name = re.sub(
+ r"\bSoldo\b|\s*-\s*", "", pl_account_name, flags=re.IGNORECASE).strip()
+ if revised_pl_account_name not in nominal:
+ errors.append(
+ f"Row {index}: 'AccountNumber' must match '{revised_pl_account_name}' in "
+ f"'NominalAnalysisNominalAnalysisNarrative/1', but found '{nominal}'."
+ )
return errors
+ @staticmethod
+ def get_account_name_from_code(account_code: str| None) -> str | None:
+ """Get the account name from the PL Account Codes table."""
+ if account_code is None:
+ return None
+ try:
+ return MeoValidSuppliers.objects.using("meo").get(
+ supplier_account_number=account_code).supplier_account_name
+ except MeoValidSuppliers.DoesNotExist:
+ return None
def _validate_nc_cc_dep_combination_against_meo_sage_account(self, data: list[dict]) -> list[str]:
"""Validate that all nominal analysis fields exist in MEO.
@@ -208,6 +231,8 @@ class CSVUploadForm(forms.Form):
max_repeat = self._get_max_repeat(fieldnames, "NominalAnalysisNominalCostCentre")
for index, row in enumerate(data, start=1):
+ claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+ claim_number = row.get("SecondReference")
for repeat in range(1, max_repeat + 1):
cc_field = f"NominalAnalysisNominalCostCentre/{repeat}"
dep_field = f"NominalAnalysisNominalDepartment/{repeat}"
@@ -241,7 +266,28 @@ class CSVUploadForm(forms.Form):
errors.append(
f"Row {index}: The combination of '{cc_field}' ({cc}), "
f"'{dep_field}' ({dep}), and '{nominal_account_field}' "
- f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
+ f"({nc}) for claimant '{claimant_name}' and claim number '{claim_number}' "
+ f"does not exist in MEO valid Sage accounts."
)
return errors
+
+ def _cheque_fields_must_be_empty(self, data: list[dict]) -> list[str]:
+ """Validate that cheque fields are empty.
+
+ The cheque fields are 'ChequeCurrencyName', 'ChequeToBankExchangeRate', and 'ChequeValueInChequeCurrency'.
+ """
+ errors = []
+ for index, row in enumerate(data, start=1):
+ cheque_currency_name = row.get("ChequeCurrencyName")
+ cheque_to_bank_exchange_rate = row.get("ChequeToBankExchangeRate")
+ cheque_value_in_cheque_currency = row.get("ChequeValueInChequeCurrency")
+ claimant_name = self.get_account_name_from_code(row.get("AccountNumber"))
+ claim_number = row.get("SecondReference")
+ if any([cheque_currency_name, cheque_to_bank_exchange_rate, cheque_value_in_cheque_currency]):
+ errors.append(
+ f"Row {index}: Unexpected values in the Cheque columns for {claimant_name} with claim number: "
+ f"{claim_number}. All cheque columns must be empty."
+ )
+
+ return errors
diff --git a/sage_validation/file_validator/models.py b/sage_validation/file_validator/models.py
index 99c2e0f7485ede4bea0112eb3bac3b46630f1676..c2bd157589984bc6b80d66d8a0bc69b9e9d70ba4 100644
--- a/sage_validation/file_validator/models.py
+++ b/sage_validation/file_validator/models.py
@@ -38,6 +38,7 @@ class MeoNominal(models.Model):
class MeoValidSageAccounts(models.Model):
"""View for MEO valid Sage accounts."""
+ id = models.UUIDField(db_column="ID", primary_key=True)
account_name = models.CharField(db_column="AccountName", max_length=60)
account_number = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
account_cost_centre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)
diff --git a/sage_validation/file_validator/templates/upload.html b/sage_validation/file_validator/templates/upload.html
index a9428905a06c54a7c936c2d18c6bcddee3c6d32a..6ea180792d0f747d9cc090265a91cafe7a9f18bc 100644
--- a/sage_validation/file_validator/templates/upload.html
+++ b/sage_validation/file_validator/templates/upload.html
@@ -3,7 +3,7 @@
{% block title %}File Upload{% endblock %}
{% block content %}
- <div class="bg-white p-10 rounded-lg shadow-lg w-11/12 md:w-1/2 lg:w-1/3 mx-auto">
+ <div class="bg-white p-10 rounded-lg shadow-lg w-11/12 md:w-3/4 lg:w-3/4 mx-auto">
<h2 class="text-2xl font-bold mb-6 text-gray-800 text-center">Upload CSV File</h2>
<form id="uploadForm" enctype="multipart/form-data" class="space-y-6">
@@ -13,10 +13,21 @@
class="w-full border border-gray-300 p-3 rounded-lg focus:outline-none focus:ring focus:ring-blue-400">
</div>
- <button type="submit"
+ <button id="submitBtn" type="submit"
class="w-full bg-blue-600 text-white py-3 rounded-lg hover:bg-blue-700 focus:outline-none focus:ring focus:ring-blue-400">
Upload
</button>
+
+ <!-- Loading Spinner -->
+ <div id="loadingSpinner" class="hidden flex justify-center">
+ <svg class="animate-spin h-6 w-6 text-blue-600" xmlns="http://www.w3.org/2000/svg" fill="none"
+ viewBox="0 0 24 24">
+ <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor"
+ stroke-width="4"></circle>
+ <path class="opacity-75" fill="currentColor"
+ d="M4 12a8 8 0 018-8v4a4 4 0 00-4 4H4z"/>
+ </svg>
+ </div>
</form>
<!-- Error Display -->
@@ -45,6 +56,8 @@
e.preventDefault();
const fileInput = document.getElementById('fileInput');
+ const submitBtn = document.getElementById('submitBtn');
+ const loadingSpinner = document.getElementById('loadingSpinner');
const errorSection = document.getElementById('errorSection');
const errorList = document.getElementById('errorList');
const successSection = document.getElementById('successSection');
@@ -58,6 +71,9 @@
successSection.classList.add('hidden');
downloadSection.classList.add('hidden');
+ submitBtn.disabled = true;
+ loadingSpinner.classList.remove('hidden');
+
const formData = new FormData();
formData.append('file', fileInput.files[0]);
@@ -85,10 +101,10 @@
li.textContent = 'You are not authorized to perform this action.';
errorList.appendChild(li);
} else if (response.status === 400 && result.status === 'error') {
- for (const [field, messages] of Object.entries(result.errors)) {
+ for (const messages of Object.values(result.errors)) {
messages.forEach(message => {
const li = document.createElement('li');
- li.textContent = `${field}: ${message}`;
+ li.textContent = message;
errorList.appendChild(li);
});
}
@@ -105,6 +121,9 @@
li.textContent = 'Failed to connect to the server. Please check your internet connection.';
errorList.appendChild(li);
errorSection.classList.remove('hidden');
+ } finally {
+ loadingSpinner.classList.add('hidden');
+ submitBtn.disabled = false;
}
});
</script>
diff --git a/setup.py b/setup.py
index 1dfbad104ea1c331d62789edcfe596035d05dcbd..d7bc1fd9dd0ff8491cda2e4eddbaa9405a62dfbf 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ from setuptools import find_packages, setup
setup(
name="sage-validation",
- version="0.6",
+ version="0.7",
packages=find_packages(),
include_package_data=True,
install_requires=[
diff --git a/test/conftest.py b/test/conftest.py
index 94a62ed9ada92e49af102a8213b29e3866eb9b90..1207568f8954c9f8cff2a5dfa862ff356cd880f8 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -37,6 +37,9 @@ def sample_input_file() -> SimpleUploadedFile:
"SYSTraderGenerationReasonType",
"GoodsValueInBaseCurrency",
"TransactionReference",
+ "ChequeCurrencyName",
+ "ChequeToBankExchangeRate",
+ "ChequeValueInChequeCurrency",
# NominalAnalysis repeating columns (Example: /1 for first occurrence)
"NominalAnalysisTransactionValue/1",
@@ -79,6 +82,9 @@ def sample_input_file() -> SimpleUploadedFile:
"1000", # SYSTraderGenerationReasonType
"1200", # GoodsValueInBaseCurrency
"BK123", # TransactionReference(Batch Number)
+ "", # ChequeCurrencyName
+ "", # ChequeToBankExchangeRate
+ "", # ChequeValueInChequeCurrency
# NominalAnalysis repeating values (Example: /1)
"500.75", # NominalAnalysisTransactionValue/1
diff --git a/test/test_file_validator/test_forms.py b/test/test_file_validator/test_forms.py
index b5df1089c1430e9a898b62cd8449ef849415f422..d32963027934d3464746ed22734d87b7d737739a 100644
--- a/test/test_file_validator/test_forms.py
+++ b/test/test_file_validator/test_forms.py
@@ -61,8 +61,8 @@ def test_source_and_trader_type_validation(sample_input_file: SimpleUploadedFile
modified_file = create_modified_csv(sample_input_file, {"Source": "90", "SYSTraderTranType": "5"})
form = CSVUploadForm(files={"file": modified_file})
assert not form.is_valid()
- assert "Row 1: 'Source' must be 80" in form.errors["file"][0]
- assert "Row 1: 'SYSTraderTranType' must be 4" in form.errors["file"][1]
+ assert "'Source' must be 80" in form.errors["file"][0]
+ assert "'SYSTraderTranType' must be 4" in form.errors["file"][1]
def test_validate_nominal_analysis_account(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
@@ -92,3 +92,11 @@ def test_validate_nc_cc_dep_combination_against_meo_sage_account(
assert ("Row 1: 'NominalAnalysisNominalCostCentre/1' (Invalid_CC) is not a valid cost centre."
in str(form.errors["file"][0]))
+
+def test_cheque_fields_must_be_empty(sample_input_file: SimpleUploadedFile, mock_meo_database: MagicMock) -> None:
+ """Test that cheque fields must be empty."""
+ modified_file = create_modified_csv(sample_input_file,
+ {"ChequeCurrencyName": "USD", "ChequeToBankExchangeRate": "1"})
+ form = CSVUploadForm(files={"file": modified_file})
+ assert not form.is_valid()
+ assert "All cheque columns must be empty." in form.errors["file"][0]