diff --git a/Changelog.md b/Changelog.md
index db0c489bdf991d3207861f10cc9b121e4ec3407e..abf10fcfcb23ac39ed7d51ac1655acb3b4b52ace 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,5 +1,8 @@
# Changelog
+## [0.3] - 2025-02-13
+- Added CSV Export and Modify Functionality
+
## [0.2] - 2025-01-27
- Added new validations
diff --git a/sage_validation/file_validator/forms.py b/sage_validation/file_validator/forms.py
index e3f9850e13b5a1799ed1d344ad354363044152f1..6c243d2efe673bb8a4fe661a1852dfa6bb7cf776 100644
--- a/sage_validation/file_validator/forms.py
+++ b/sage_validation/file_validator/forms.py
@@ -1,13 +1,19 @@
"""Forms for the file_validator app."""
import csv
-from collections.abc import Iterable, Sequence
+from collections.abc import Sequence
from typing import ClassVar
from django import forms
from django.core.files.uploadedfile import UploadedFile
-from sage_validation.file_validator.models import Meocostcentres, Meovalidsageaccounts, PlAccountCodes, XxData
+from sage_validation.file_validator.models import (
+ MeoCostCentres,
+ MeoNominal,
+ MeoValidSageAccounts,
+ MeoValidSuppliers,
+ XxData,
+)
class CSVUploadForm(forms.Form):
@@ -72,7 +78,6 @@ class CSVUploadForm(forms.Form):
self._validate_headers(fieldnames)
error_list = []
- # Step 3: Validate 'Source' and 'SYSTraderTranType' values
data = list(reader)
error_list.extend(self._validate_source_and_trader_type(data))
error_list.extend(self._validate_nominal_analysis_account(data))
@@ -121,7 +126,7 @@ class CSVUploadForm(forms.Form):
raise forms.ValidationError(msg)
@staticmethod
- def _validate_source_and_trader_type(data: Iterable[dict]) -> list:
+ def _validate_source_and_trader_type(data: list[dict]) -> list:
"""Validate that 'Source' is always 80 and 'SYSTraderTranType' is always 4."""
errors = []
@@ -135,14 +140,14 @@ class CSVUploadForm(forms.Form):
return errors
@staticmethod
- def _validate_nominal_analysis_account(data: Iterable[dict]) -> list[str]:
+ def _validate_nominal_analysis_account(data: list[dict]) -> list[str]:
"""Validate that 'AccountNumber' matches the name in 'NominalAnalysisNominalAnalysisNarrative/1'.
This only checks the first group of NominalAnalysis columns. A list of codes/names
is fetched from the database for validation (from the 'PL Account Codes' table).
Args:
- data (Iterable[dict]): The rows of data to validate.
+ data (list[dict]): The rows of data to validate.
Returns:
List[str]: A list of error messages, if any.
@@ -151,8 +156,8 @@ class CSVUploadForm(forms.Form):
errors = []
account_code_map = {
- obj.pl_account_code: obj.pl_account_name
- for obj in PlAccountCodes.objects.using("meo").all() # type: ignore[attr-defined]
+ obj.supplier_account_number: obj.supplier_account_name
+ for obj in MeoValidSuppliers.objects.using("meo").all() # type: ignore[attr-defined]
}
for index, row in enumerate(data, start=1):
@@ -174,12 +179,15 @@ class CSVUploadForm(forms.Form):
return errors
- @staticmethod
- def _validate_nc_cc_dep_combination_against_meo_sage_account(data: Iterable[dict]) -> list[str]:
- """Validate that the combination of 'AccountCostCentre', 'AccountDepartment', and 'AccountNumber' exists in MEO.
+
+ def _validate_nc_cc_dep_combination_against_meo_sage_account(self, data: list[dict]) -> list[str]:
+ """Validate that all nominal analysis fields exist in MEO.
+
+ This includes 'NominalAnalysisNominalCostCentre/{N}', 'NominalAnalysisNominalDepartment/{N}',
+ and 'NominalAnalysisNominalAccountNumber/{N}'.
Args:
- data (Iterable[dict]): The rows of data to validate.
+ data (list[dict]): The rows of data to validate.
Returns:
List[str]: A list of error messages, if any.
@@ -188,41 +196,52 @@ class CSVUploadForm(forms.Form):
errors = []
cost_centre_map = {
- obj.cc: obj.cctype for obj in Meocostcentres.objects.using("meo").all()
+ obj.cc: obj.cc_type for obj in MeoCostCentres.objects.using("meo").all()
}
xx_data_map = {
obj.xx_value: (obj.project, obj.overhead) for obj in XxData.objects.using("meo").all()
}
+
+ fieldnames = list(data[0].keys())
+ max_repeat = self._get_max_repeat(fieldnames, "NominalAnalysisNominalCostCentre")
+
for index, row in enumerate(data, start=1):
- cc = row.get("NominalAnalysisNominalCostCentre/1")
- dep = row.get("NominalAnalysisNominalDepartment/1")
- nominal_account_name = row.get("NominalAnalysisNominalAccountNumber/1")
+ for repeat in range(1, max_repeat + 1):
+ cc_field = f"NominalAnalysisNominalCostCentre/{repeat}"
+ dep_field = f"NominalAnalysisNominalDepartment/{repeat}"
+ nominal_account_field = f"NominalAnalysisNominalAccountNumber/{repeat}"
- if not cc or not dep or not nominal_account_name:
- continue
+ cc = row.get(cc_field)
+ dep = row.get(dep_field)
+ nominal_account_name = row.get(nominal_account_field)
- cc_type = cost_centre_map.get(cc)
- if not cc_type:
- errors.append(f"Row {index}: 'NominalAnalysisNominalCostCentre/1' ({cc}) is not a valid cost centre.")
- continue
+ if not cc or not dep or not nominal_account_name:
+ continue
- xx_data = xx_data_map.get(nominal_account_name)
- if not xx_data:
- errors.append(
- f"Row {index}: 'NominalAnalysisNominalAccountNumber/1' ({nominal_account_name}) is not valid.")
- continue
+ cc_type = cost_centre_map.get(cc)
+ if not cc_type:
+ errors.append(f"Row {index}: '{cc_field}' ({cc}) is not a valid cost centre.")
+ continue
- nc = xx_data[0] if cc_type == "Project" else xx_data[1]
+ xx_data = xx_data_map.get(nominal_account_name)
- if not Meovalidsageaccounts.objects.using("meo").filter(
- accountcostcentre=cc, accountdepartment=dep, accountnumber=nc
- ).exists():
- errors.append(
- f"Row {index}: The combination of 'NominalAnalysisNominalCostCentre/1' ({cc}), "
- f"'NominalAnalysisNominalDepartment/1' ({dep}), and 'NominalAnalysisNominalAccountNumber/1' "
- f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
- )
+ if xx_data:
+ nc = xx_data[0] if cc_type == "Project" else xx_data[1]
+ elif MeoNominal.objects.using("meo").filter(nom=nominal_account_name).exists():
+ nc = nominal_account_name
+ else:
+ errors.append(f"Row {index}: '{nominal_account_field}' ({nominal_account_name}) is not valid.")
+ continue
+
+ if not MeoValidSageAccounts.objects.using("meo").filter(
+ account_cost_centre=cc, account_department=dep, account_number=nc
+ ).exists():
+ errors.append(
+ f"Row {index}: The combination of '{cc_field}' ({cc}), "
+ f"'{dep_field}' ({dep}), and '{nominal_account_field}' "
+ f"({nominal_account_name}) does not exist in MEO valid Sage accounts."
+ )
return errors
diff --git a/sage_validation/file_validator/models.py b/sage_validation/file_validator/models.py
index 0bc48ec3eb16cc8dad934cb145068fb76e8935ab..99c2e0f7485ede4bea0112eb3bac3b46630f1676 100644
--- a/sage_validation/file_validator/models.py
+++ b/sage_validation/file_validator/models.py
@@ -2,24 +2,7 @@
from django.db import models
-class PlAccountCodes(models.Model):
- """Represents profit and loss (PL) account codes.
-
- This model maps the PL account names to their respective codes.
- """
-
- objects = None
- pl_account_name = models.TextField(db_column="PL_Account_Name")
- pl_account_code = models.CharField(db_column="PL_Account_Code", primary_key=True, max_length=50)
-
- class Meta:
- """Metaclass for the PlAccountCodes model."""
-
- managed = False
- db_table = "PL_Account_Codes"
-
-
-class Meocostcentres(models.Model):
+class MeoCostCentres(models.Model):
"""Represents cost centres.
This model contains data related to cost centres,
@@ -28,68 +11,68 @@ class Meocostcentres(models.Model):
cc = models.CharField(db_column="CC", max_length=3)
cc_name = models.CharField(db_column="CC_Name", max_length=50, blank=True, null=True)
- cctype = models.CharField(db_column="CCType", max_length=8)
+ cc_type = models.CharField(db_column="CCType", max_length=8)
id = models.IntegerField(db_column="ID", primary_key=True)
class Meta:
- """Metaclass for the Meocostcentres model."""
+ """Metaclass for the MeoCostCentres model."""
managed = False
db_table = "meoCostCentres"
-class Meonominal(models.Model):
+class MeoNominal(models.Model):
"""View for MEO nominal codes."""
nom = models.CharField(db_column="Nom", max_length=5)
- nomname = models.CharField(db_column="NomName", max_length=60, blank=True, null=True)
- nomid = models.IntegerField(db_column="NomID")
+ nom_name = models.CharField(db_column="NomName", max_length=60, blank=True, null=True)
+ nom_id = models.IntegerField(db_column="NomID")
class Meta:
- """Metaclass for the Meonominal model."""
+ """Metaclass for the MeoNominal model."""
managed = False
db_table = "meoNominal"
-class Meovalidsageaccounts(models.Model):
+class MeoValidSageAccounts(models.Model):
"""View for MEO valid Sage accounts."""
- accountname = models.CharField(db_column="AccountName", max_length=60)
- accountnumber = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
- accountcostcentre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)
- accountdepartment = models.CharField(db_column="AccountDepartment", max_length=3, blank=True, null=True)
+ account_name = models.CharField(db_column="AccountName", max_length=60)
+ account_number = models.CharField(db_column="AccountNumber", max_length=8, blank=True, null=True)
+ account_cost_centre = models.CharField(db_column="AccountCostCentre", max_length=3, blank=True, null=True)
+ account_department = models.CharField(db_column="AccountDepartment", max_length=3, blank=True, null=True)
class Meta:
- """Metaclass for the Meovalidsageaccounts model."""
+ """Metaclass for the MeoValidSageAccounts model."""
managed = False
db_table = "meoValidSageAccounts"
-class Meovalidsuppliers(models.Model):
+class MeoValidSuppliers(models.Model):
"""View for MEO valid suppliers."""
- supplieraccountnumber = models.CharField(db_column="SupplierAccountNumber", max_length=8)
- supplieraccountname = models.CharField(db_column="SupplierAccountName", max_length=60)
+ supplier_account_number = models.CharField(db_column="SupplierAccountNumber", max_length=8, primary_key=True)
+ supplier_account_name = models.CharField(db_column="SupplierAccountName", max_length=60)
class Meta:
- """Metaclass for the Meovalidsuppliers model."""
+ """Metaclass for the MeoValidSuppliers model."""
managed = False
db_table = "meoValidSuppliers"
-class Meovalidvat(models.Model):
+class MeoValidVat(models.Model):
"""View for MEO valid VAT codes."""
tax_code = models.SmallIntegerField(db_column="Tax code")
tax_code_label = models.CharField(db_column="tax code label", max_length=60)
rate = models.DecimalField(db_column="Rate", max_digits=18, decimal_places=2)
- inputnominalaccount = models.CharField(db_column="InputNominalAccount", max_length=8, blank=True, null=True)
+ input_nominal_account = models.CharField(db_column="InputNominalAccount", max_length=8, blank=True, null=True)
class Meta:
- """Metaclass for the Meovalidvat model."""
+ """Metaclass for the MeoValidVat model."""
managed = False
db_table = "meoValidVAT"
diff --git a/sage_validation/file_validator/templates/upload.html b/sage_validation/file_validator/templates/upload.html
index 129121ef05336a83485f0533f17cf4cac7afc749..4a595da4794ec9281508073a490bc54daa4549f5 100644
--- a/sage_validation/file_validator/templates/upload.html
+++ b/sage_validation/file_validator/templates/upload.html
@@ -26,12 +26,21 @@
</div>
<!-- Success Message -->
- <div id="successSection" class="hidden mt-4 bg-green-100 border border-green-400 text-green-700 p-4 rounded-lg">
- <strong class="font-bold">Success:</strong>
- <span id="successMessage" class="block mt-2"></span>
+ <div id="successSection"
+ class="hidden mt-4 mb-8 bg-green-100 border border-green-400 text-green-700 p-4 rounded-lg text-center">
+ <strong class="font-bold text-lg">Success!</strong>
+ <p id="successMessage" class="block mt-2 text-md"></p>
+ </div>
+ <div id="downloadSection" class="hidden mt-16 flex justify-center">
+ <a id="downloadLink" href="#"
+ class="w-full max-w-xs text-center bg-blue-600 text-white py-4 px-12 rounded-lg shadow-lg hover:bg-blue-700
+ focus:outline-none focus:ring focus:ring-blue-400 transition text-lg font-semibold block">
+ 📥 Download Updated CSV
+ </a>
</div>
</div>
+
<script>
const form = document.getElementById('uploadForm');
const fileInput = document.getElementById('fileInput');
@@ -39,6 +48,8 @@
const errorList = document.getElementById('errorList');
const successSection = document.getElementById('successSection');
const successMessage = document.getElementById('successMessage');
+ const downloadSection = document.getElementById('downloadSection');
+ const downloadLink = document.getElementById('downloadLink');
form.addEventListener('submit', async function (e) {
e.preventDefault();
@@ -48,6 +59,7 @@
successMessage.innerHTML = '';
errorSection.classList.add('hidden');
successSection.classList.add('hidden');
+ downloadSection.classList.add('hidden');
const formData = new FormData();
formData.append('file', fileInput.files[0]);
@@ -66,18 +78,35 @@
if (response.ok && result.status === 'success') {
successMessage.innerText = result.message;
successSection.classList.remove('hidden');
+ downloadLink.href = result.download_url;
+ downloadSection.classList.remove('hidden');
} else if (response.status === 400 && result.status === 'error') {
- // Handle form errors from the backend
+ errorList.innerHTML = '';
+
if (Array.isArray(result.errors)) {
result.errors.forEach(errorObj => {
- for (const [field, messages] of Object.entries(errorObj)) {
- messages.forEach(message => {
- const li = document.createElement('li');
- li.textContent = `${field}: ${message}`;
- errorList.appendChild(li);
- });
+ if (typeof errorObj === 'string') {
+ const li = document.createElement('li');
+ li.textContent = errorObj;
+ errorList.appendChild(li);
+ } else {
+ for (const [field, messages] of Object.entries(errorObj)) {
+ messages.forEach(message => {
+ const li = document.createElement('li');
+ li.textContent = `${field}: ${message}`;
+ errorList.appendChild(li);
+ });
+ }
}
});
+ } else if (typeof result.errors === 'object') {
+ for (const [field, messages] of Object.entries(result.errors)) {
+ messages.forEach(message => {
+ const li = document.createElement('li');
+ li.textContent = `${field}: ${message}`;
+ errorList.appendChild(li);
+ });
+ }
} else {
const li = document.createElement('li');
li.textContent = result.errors;
@@ -87,7 +116,6 @@
errorSection.classList.remove('hidden');
}
} catch (error) {
- // Handle unexpected errors
const li = document.createElement('li');
li.textContent = 'An unexpected error occurred. Please try again.';
errorList.appendChild(li);
diff --git a/sage_validation/file_validator/urls.py b/sage_validation/file_validator/urls.py
index 12e1520494e4f3bda7dfb0b9e7b2dd8b0ead954a..a5c0d9fd7840097233aabed1184a392037d7bcc1 100644
--- a/sage_validation/file_validator/urls.py
+++ b/sage_validation/file_validator/urls.py
@@ -2,8 +2,9 @@
from django.urls import path
-from sage_validation.file_validator.views import CSVUploadView
+from sage_validation.file_validator.views import CSVExportView, CSVUploadView
urlpatterns = [
path("upload/", CSVUploadView.as_view(), name="upload-file"),
+ path("export/", CSVExportView.as_view(), name="export-file"),
]
diff --git a/sage_validation/file_validator/views.py b/sage_validation/file_validator/views.py
index 33ec72f6cc93b6db8ad5aafa505629af0e2d6677..99efd175b0fa9498eb15127578cd85e272e21ac1 100644
--- a/sage_validation/file_validator/views.py
+++ b/sage_validation/file_validator/views.py
@@ -1,11 +1,17 @@
"""Views for the file_validator app."""
+import csv
+import io
+from typing import Any
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import render
from django.urls import reverse_lazy
+from django.utils import timezone
+from django.views.generic.base import View
from django.views.generic.edit import FormView
from sage_validation.file_validator.forms import CSVUploadForm
+from sage_validation.file_validator.models import MeoCostCentres, XxData
def index_view(request: HttpRequest) -> HttpResponse:
@@ -20,17 +26,92 @@ class CSVUploadView(FormView):
form_class = CSVUploadForm
success_url = reverse_lazy("upload-file")
- def get_context_data(self, **kwargs: dict) -> dict:
+ def get_context_data(self, **kwargs: dict[str, Any]) -> dict[str, Any]:
"""Render the form with no error message on GET request."""
context = super().get_context_data(**kwargs)
context["error"] = None
context["message"] = None
return context
- def form_valid(self, form: CSVUploadForm) -> JsonResponse: # noqa: ARG002
- """Handle the CSV validation and passes appropriate success messages to the template."""
- return JsonResponse({"status": "success", "message": "File is valid"})
+ def form_valid(self, form: CSVUploadForm) -> JsonResponse:
+ """Handle the CSV validation, store valid data, and prepare for export."""
+ csv_file = form.cleaned_data["file"]
+ csv_file.seek(0)
+ decoded_file = csv_file.read().decode("utf-8").strip()
+
+ if not decoded_file:
+ return JsonResponse({"status": "error", "message": "Uploaded file is empty."}, status=400)
+
+ reader = csv.DictReader(io.StringIO(decoded_file))
+ csv_data: list[dict[str, str]] = list(reader)
+
+ updated_data = self.update_fields(csv_data)
+
+ self.request.session["validated_csv"] = updated_data
+ self.request.session.modified = True
+
+ return JsonResponse({
+ "status": "success",
+ "message": "File successfully uploaded and processed.",
+ "download_url": reverse_lazy("export-file")
+ })
def form_invalid(self, form: CSVUploadForm) -> JsonResponse:
- """Handle the form when it is invalid (e.g., wrong file type or validation errors)."""
- return JsonResponse({"status": "error", "errors": [form.errors]}, status=400)
+ """Handle the form when it is invalid."""
+ return JsonResponse({"status": "error", "errors": form.errors}, status=400)
+
+ @staticmethod
+ def update_fields(csv_data: list[dict[str, str]]) -> list[dict[str, str]]:
+ """Automatically update specific fields before export."""
+ current_date: str = timezone.now().strftime("%d/%m/%Y")
+
+ xx_data_map: dict[str, tuple] = {
+ obj.xx_value: (obj.project, obj.overhead) for obj in XxData.objects.using("meo").all()
+ }
+ cost_centre_map: dict[str, str] = {
+ obj.cc: obj.cc_type for obj in MeoCostCentres.objects.using("meo").all()
+ }
+
+ for row in csv_data:
+ row["TransactionDate"] = current_date
+
+ repeat = 1
+ while f"NominalAnalysisNominalCostCentre/{repeat}" in row:
+ cc = row.get(f"NominalAnalysisNominalCostCentre/{repeat}", "")
+ nominal_account_name = row.get(f"NominalAnalysisNominalAccountNumber/{repeat}", "")
+
+ if not cc or not nominal_account_name:
+ repeat += 1
+ continue
+
+ cc_type = cost_centre_map.get(cc, "")
+ xx_data = xx_data_map.get(nominal_account_name)
+
+ if xx_data:
+ row[f"NominalAnalysisNominalAccountNumber/{repeat}"] = (
+ xx_data[0] if cc_type == "Project" else xx_data[1]
+ )
+
+ repeat += 1
+
+ return csv_data
+
+
+class CSVExportView(View):
+ """View for exporting the updated CSV file."""
+
+ def get(self, request: HttpRequest) -> HttpResponse:
+ """Generate a downloadable CSV file with updated values."""
+ csv_data: list[dict[str, str]] = request.session.get("validated_csv", [])
+
+ if not csv_data:
+ return HttpResponse("No data available for export.", status=400)
+
+ response = HttpResponse(content_type="text/csv")
+ response["Content-Disposition"] = 'attachment; filename="updated_file.csv"'
+
+ writer = csv.DictWriter(response, fieldnames=csv_data[0].keys())
+ writer.writeheader()
+ writer.writerows(csv_data)
+
+ return response
diff --git a/sage_validation/settings.py b/sage_validation/settings.py
index 9d508100b933598577dc26f8450229d7f1e13bcf..1a00d60d4de5129a8f1a28d43bc7160f5cea9272 100644
--- a/sage_validation/settings.py
+++ b/sage_validation/settings.py
@@ -89,6 +89,8 @@ DATABASES = {
'OPTIONS': {
'driver': 'ODBC Driver 18 for SQL Server',
+ 'extra_params': 'TrustServerCertificate=Yes;'
+
},
},
}
diff --git a/setup.py b/setup.py
index bb1bd54466c3270083abeb5acc7b0dac98de5064..09c3d78cb76669e38b66c2433c81b9f3ece4497f 100644
--- a/setup.py
+++ b/setup.py
@@ -3,12 +3,12 @@ from setuptools import find_packages, setup
setup(
name="sage-validation",
- version="0.3",
+ version="0.4",
packages=find_packages(),
include_package_data=True,
install_requires=[
"Django==5.0.11",
- "mssql-django",
+ "mssql-django==1.5",
],
extras_require={
"prod": [
diff --git a/tox.ini b/tox.ini
index 74a6ea7a599a22889bc9271e231206a165c7cafb..91d0161e6590165ecf51bec47ab84ff68718aacd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py312
+envlist = py311
[testenv]
deps =